From 5992ef05f82abd17c6731f9c2328ffe889c1b180 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Fri, 2 Oct 2026 18:50:19 +0000 Subject: [PATCH 01/10] feat(routing): publish and serve authoritative no-route revisions Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 29 +- charts/praxis-gateway/README.md | 16 + deploy/crds/gridnetwork.yaml | 29 +- docs/architecture/consumer-config.md | 124 +- docs/architecture/crds.md | 60 +- gateway/Cargo.lock | 16 +- gateway/ai-grid-filters/src/control.rs | 16 +- gateway/ai-grid-filters/src/descriptor.rs | 68 +- gateway/ai-grid-filters/src/flow.rs | 1 + gateway/ai-grid-filters/src/route.rs | 8 + gateway/ai-grid-filters/src/serving.rs | 4 + gateway/ai-grid-filters/src/snapshot.rs | 18 + operator/Cargo.toml | 2 +- operator/src/controller/grid_network.rs | 165 +- operator/src/controller/inference_provider.rs | 45 +- operator/src/crd/grid_network.rs | 88 +- operator/src/resources/consumer_config.rs | 393 +- operator/src/resources/routing_overlay.rs | 7 +- operator/src/resources/serving_config.rs | 68 +- overlay-sync/src/watcher.rs | 58 + .../grid-provider-traffic/README.md | 39 +- .../grid-provider-traffic/forge.yaml | 6 + .../common/vcr-provider-workload.yaml | 28 + xtask/src/env/certs.rs | 42 +- xtask/src/env/forge_config.rs | 35 +- xtask/src/env/image_overrides.rs | 41 +- .../src/env/provider_traffic_qualification.rs | 3373 +++++++++++++++-- .../src/env/static_weighted_qualification.rs | 466 ++- 28 files changed, 4516 insertions(+), 729 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index a9fb94d50..29aa1e050 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -178,7 +178,9 @@ spec: entry per unique inference cluster. Other capability kinds remain in the routing overlay for dedicated data-plane pipelines. - The generated `ConfigMap` contains no token bytes. + The generated `ConfigMap` contains no token bytes. Credential-bearing + overlays are withheld until the consumer has explicitly declared that its + compatible filter config is already running. nullable: true properties: clusterEndpoints: @@ -290,11 +292,23 @@ spec: description: |- Base directory for mounted credential Secret files inside the consumer pod. - Each credential Secret is expected to be mounted at - `{credentialMountBase}/{secret-name}/{secret-key}`. + In projected-credential mode, mount each Secret at + `{credentialMountBase}/{secret-namespace}/{secret-name}` with its data + keys as files. Static `file:` entries continue to use their explicit + paths and are not constrained by this directory layout. Default: `/run/secrets/grid-credentials`. type: string + enableProjectedCredentials: + default: false + description: |- + Generate the dynamic `credential_inject` filter and projected mount + root, including when the current overlay has no credentials. Enable this + first, then roll out the generated config before asserting readiness + with `supportsProjectedCredentials`. + + Default: `false`. + type: boolean enabled: default: false description: |- @@ -435,6 +449,15 @@ spec: namespace: description: Gateway namespace. type: string + supportsEmptyOverlay: + default: false + description: |- + Explicitly attest that every consumer of this gateway's routing overlay + accepts a valid versioned snapshot with `candidates: []` as a no-route + decision. Defaults to `false` for compatibility with released Praxis + images that reject empty candidate lists. The operator retains the last + distributed revision until this is enabled. + type: boolean required: - name - namespace diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index b992cd156..753b66d04 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -397,6 +397,22 @@ balancing and includes Basic Auth. It does not include the optional qualification. That qualification is not supported by this default image; AGN does not publish a replacement AI rollup. +The default image also predates empty versioned routing snapshots. Keep +`GridNetwork.spec.gatewayRefs[].supportsEmptyOverlay` unset/false when using this +default; Grid will retain the last distributed overlay rather than sending a +snapshot this image rejects. Upgrade and roll every consumer to an image with +empty-snapshot support before explicitly enabling that GridNetwork capability. + +Generated consumer credentials use a separate two-step opt-in: set +`consumerConfig.enableProjectedCredentials: true`, let Grid render the filter, +and roll out the consumer with the read-only Secret mounts. In this mode, mount +each Secret at `{credentialMountBase}/{secret-namespace}/{secret-name}`, with +its data keys as files (for example, +`/run/secrets/grid-credentials/grid-system/provider-key/token`). Static `file:` +entries continue to use their explicitly configured paths. Only then set +`consumerConfig.supportsProjectedCredentials: true`; Grid retains credential- +bearing overlays until that readiness attestation is present. + ### Edge and provider gateways in AGN AGN runs this chart in two roles with different values: diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 4fa0c2fc1..a52c365dc 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -170,7 +170,9 @@ spec: entry per unique inference cluster. Other capability kinds remain in the routing overlay for dedicated data-plane pipelines. - The generated `ConfigMap` contains no token bytes. + The generated `ConfigMap` contains no token bytes. Credential-bearing + overlays are withheld until the consumer has explicitly declared that its + compatible filter config is already running. nullable: true properties: clusterEndpoints: @@ -282,11 +284,23 @@ spec: description: |- Base directory for mounted credential Secret files inside the consumer pod. - Each credential Secret is expected to be mounted at - `{credentialMountBase}/{secret-name}/{secret-key}`. + In projected-credential mode, mount each Secret at + `{credentialMountBase}/{secret-namespace}/{secret-name}` with its data + keys as files. Static `file:` entries continue to use their explicit + paths and are not constrained by this directory layout. Default: `/run/secrets/grid-credentials`. type: string + enableProjectedCredentials: + default: false + description: |- + Generate the dynamic `credential_inject` filter and projected mount + root, including when the current overlay has no credentials. Enable this + first, then roll out the generated config before asserting readiness + with `supportsProjectedCredentials`. + + Default: `false`. + type: boolean enabled: default: false description: |- @@ -427,6 +441,15 @@ spec: namespace: description: Gateway namespace. type: string + supportsEmptyOverlay: + default: false + description: |- + Explicitly attest that every consumer of this gateway's routing overlay + accepts a valid versioned snapshot with `candidates: []` as a no-route + decision. Defaults to `false` for compatibility with released Praxis + images that reject empty candidate lists. The operator retains the last + distributed revision until this is enabled. + type: boolean required: - name - namespace diff --git a/docs/architecture/consumer-config.md b/docs/architecture/consumer-config.md index 8ff38c253..65387fefc 100644 --- a/docs/architecture/consumer-config.md +++ b/docs/architecture/consumer-config.md @@ -86,15 +86,34 @@ confirms all 9 routing assertions pass with the live consumer pod running the operator-generated config. Token bytes are absent from the `ConfigMap`, consumer-cluster replica, overlay JSON, and all logs. -The generated config is a complete, runnable Praxis config containing: +The generated config is a complete Praxis config whose route state is read from +the same versioned overlay ConfigMap Grid publishes for the gateway. The +consumer Deployment must mount that ConfigMap's `routing-overlay.json` key at +`/etc/praxis/routing/routing-overlay.json` as a projected volume (not with +`subPath`) and mount the generated `praxis.yaml` as its Praxis config. For a +cross-cluster consumer, both ConfigMaps must be delivered to the consumer +cluster. The initial migration from previously generated inline candidates +requires a consumer rollout so the new filter configuration and volume mount +take effect. + +The generated config contains: - `listeners:` — one public listener at `0.0.0.0:{listenerPort}` (default 8080) - `filter_chains:` — the consumer filter chain: - - `intelligent_route` candidates from the overlay (with `credential.secretRef` for - credential-bearing candidates) - - `credential_inject` entries using `file:` sources when credential-bearing - candidates are present — token bytes are never written to the `ConfigMap` - - `load_balancer` entries (one per unique candidate cluster). Every referenced + - `intelligent_route` using `overlay_file`, exact network/gateway/namespace/site + scope checks, and hot reload. Candidate and selection-policy state is not + duplicated in startup-only YAML. `provider_hop_clusters` is derived from + the explicit endpoint inventory: all `mutual_tls` provider-gateway entries + are included, while plaintext local/dev endpoints are excluded. This lets + Praxis attach request context required by the provider-side `provider_route` + filter. + - a `credential_inject` filter when current routes need credentials or + `enableProjectedCredentials: true`. In projected mode it starts with an + empty table and resolves selected references under the projected mount + root. Missing or invalid files reject with HTTP 503. Token bytes are never + written to the `ConfigMap`. + - `load_balancer` entries for every configured endpoint, including currently + inactive providers needed for restoration. Every potentially routable cluster must have a matching `consumerConfig.clusterEndpoints[]` entry with endpoint address and explicit `transport` configuration (`mutual_tls`, `tls`, or `plaintext`). Missing transport fails closed: the operator will @@ -116,14 +135,90 @@ reference is rendered and mounted only on the gateway whose local site matches the candidate's site. Remote candidates therefore receive their credentials at the provider site's final backend hop, rather than at an earlier gateway. -The generated config requires a Praxis AI image that contains the -`credential_inject` filter. AGN can render the config and project -credential references today; deployments must ensure the selected Praxis AI image -includes the matching request-time filter. +Both `enableProjectedCredentials` and `supportsProjectedCredentials` default to +`false`. The first generates the filter with an empty credential table even +when there are no credential-bearing candidates. Roll out the consumer, mount +the referenced Secret, and only then set the second field as a readiness +attestation. The Grid controller does not own or restart the consumer +Deployment. Until both are true, it retains credential-bearing revisions and +reports `ProjectedCredentialsUnsupported`. With the capability enabled, later +credential-bearing revisions cannot bypass injection: Praxis resolves the +selected reference or rejects with HTTP 503. + +In projected mode, mount each Secret at +`{credentialMountBase}/{secretRef.namespace}/{secretRef.name}` with Secret data +keys as files; Praxis resolves a selected key beneath that directory. This +namespace segment prevents same-name Secrets from aliasing. Existing configured +`file:` sources keep using their explicit paths and are not changed by this +projected-mode contract. + +Example overlay mount for a same-cluster consumer (the ConfigMap name is +`grid-overlay--`, subject to the operator's deterministic +name shortening): + +```yaml +volumes: + - name: grid-routing-overlay + configMap: + name: grid-overlay-production-inference-gw + items: + - key: routing-overlay.json + path: routing-overlay.json +containers: + - name: praxis + volumeMounts: + - name: grid-routing-overlay + mountPath: /etc/praxis/routing + readOnly: true +``` + +The overlay is the live route authority: a valid empty envelope produces no +route, and malformed replacements retain Praxis's last-known-good snapshot. +Changes to listener, endpoint/TLS topology, or the generated credential +injection table still require the consumer owner to roll/reload Praxis after +the generated `praxis.yaml` changes. That rollout is separate from route-only +overlay reloads. If credentials are present, a restored provider whose +credential reference changed requires this config rollout before its request +can succeed; `credential_inject` fails closed when the configured reference +does not match the overlay. See [`docs/architecture/crds.md`](crds.md#gatewayrefconsumerconfig) for the full field reference. +## Grid-managed live-routing consumers + +Grid has three live candidate consumers. Their route state is not equivalent to +an empty list in arbitrary startup YAML: + +| Consumer path | Route-state source | Valid no-route state | Runtime update behavior | +|---|---|---|---| +| Praxis `intelligent_route` with `overlay_file` | Grid's versioned `routing-overlay.json` ConfigMap | A valid versioned envelope whose `overlay.candidates` is `[]` | Praxis validates and atomically serves the new snapshot; malformed replacements retain last-known-good. Grid distributes an empty revision only after `GatewayRef.supportsEmptyOverlay: true`. | +| Generated `GatewayRef.consumerConfig` | The same scoped versioned overlay; generated `praxis.yaml` supplies filter and endpoint plumbing | The same empty envelope | Candidate-only updates hot reload. Credential-bearing revisions are held until `enableProjectedCredentials` has been rolled out and `supportsProjectedCredentials: true` attests the filter and Secret mounts are active. Listener, endpoint/TLS, and filter-pipeline changes still require the consumer owner to reload or roll out its Praxis configuration. | +| Embedded `grid-gateway` `grid_site_route` filter | The operator-published `grid-serving--` ConfigMap | A valid serving config with `candidates: []` | The running gateway watches the projected serving file and atomically replaces its candidate snapshot and provider-hop allowlist. Malformed updates retain the previous snapshot. | + +`GatewayRef.supportsEmptyOverlay` defaults to `false`. Set it to `true` only +after every data-plane consumer of that gateway's overlay has been upgraded and +rolled out with empty-snapshot support. Until then, Grid retains the last +distributed revision and reports `EmptyOverlayUnsupported`; this is an explicit +compatibility boundary, not a successful withdrawal. + +The embedded gateway's filter chain and upstream cluster definitions remain in +its startup Praxis configuration, but its changing candidate list is not a +startup-only inline list: it comes from the watched Grid serving ConfigMap. +Provider-hop context is emitted only for clusters explicitly configured as +`mutual_tls`; the embedded gateway carries the Grid overlay's stable candidate +ID and generates a fresh hop request ID. Caller-supplied routing-context +headers are removed before forwarding. The provider gateway still authenticates +the peer with mTLS before consuming that context. + +Static, manually configured `intelligent_route.candidates` remain a distinct +contract: an empty static list is rejected, and these immutable candidate lists +do not receive Grid's runtime-withdrawal semantics. The operator's +`overlay_bridge` helper is currently a conversion/test utility, not an active +controller consumer. Grid-managed runtime withdrawal must use one of the three +live paths above; publishing an empty overlay does not change an unrelated +static Praxis configuration. + ## Operational diagnostics After enabling `consumerConfig.enabled: true` for a gateway, the `GridNetwork` @@ -287,6 +382,7 @@ Example failure output: | `MissingTransport` | `Error` | A cluster endpoint has no `transport` configuration — the operator refuses to guess TLS vs plaintext | | `MissingSni` | `Error` | A `mutual_tls` or `tls` cluster endpoint has no (or blank) `sni`; TLS requires a server name | | `PlaintextWithSni` | `Error` | A `plaintext` cluster endpoint has `sni` set — `sni` does not enable TLS; use `mutual_tls` if TLS is intended | +| `ProjectedCredentialsUnsupported` | `Error` | A credential-bearing overlay is retained until the consumer declares that its projected credential filter is already running | | `ConsumerConfigRenderFailed` | `Error` | Overlay data produced an unrenderable config (e.g. blank local site) | | `ConsumerConfigApplyFailed` | `Error` | Kubernetes API rejected the `ConfigMap` apply (e.g. RBAC, namespace not found) | | `ConsumerConfigError` | `Error` | Other error during render or apply | @@ -373,6 +469,14 @@ static endpoint/TLS topology. The intended architecture separates these: - **Dynamic overlay** (`routing-overlay.json` envelope): changes are consumable without a full restart through `intelligent_route` overlay-file hot reload. +For operator-generated consumer configs, `intelligent_route.provider_hop_clusters` +is derived from the explicit endpoint inventory: every +`clusterEndpoints[]` entry with `transport.mode: mutual_tls` is included, while +explicit plaintext local/dev endpoints are not. This ensures requests routed +through the authenticated provider-gateway hop carry the candidate and request +context required by the provider's `provider_route` filter. The setting is +retained when candidate and selection state moves into the versioned overlay. + Praxis AI validates each projected envelope before atomically replacing the in-memory route snapshot. A malformed replacement retains the same-process last-known-good snapshot. The deployment must mount the projected directory diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 397873fc7..18ff6f3c9 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -27,8 +27,11 @@ spec: - name: inference-gw namespace: praxis-system localSiteName: cluster-east # optional; defaults to network name + supportsEmptyOverlay: true # set only after every consumer accepts empty v1 snapshots consumerConfig: # optional; opt-in consumer Praxis config generation enabled: true + enableProjectedCredentials: true # generate filter, then roll out consumer + supportsProjectedCredentials: true # set only after that rollout credentialMountBase: /run/secrets/grid-credentials configMapName: praxis-consumer-config tlsCertMountPath: /etc/praxis/tls @@ -291,7 +294,9 @@ Praxis `ConfigMap` generation. | Field | Default | Meaning | |---|---|---| | `enabled` | `false` | Set to `true` to enable consumer config generation for this gateway. | -| `credentialMountBase` | `/run/secrets/grid-credentials` | Base directory where credential Secrets are mounted inside the consumer pod. | +| `enableProjectedCredentials` | `false` | Generate the dynamic `credential_inject` filter and projected mount root, including with an empty candidate set. Roll out the consumer after enabling this. | +| `supportsProjectedCredentials` | `false` | Attest that the consumer has loaded that filter/config and mounted its referenced Secrets. Until true, credential-bearing overlay revisions are retained with `ProjectedCredentialsUnsupported`. | +| `credentialMountBase` | `/run/secrets/grid-credentials` | Base directory for projected credentials. In projected mode, mount each Secret at `{base}/{namespace}/{name}` with Secret keys as files. Static `file:` entries continue using their explicitly configured paths. | | `configMapName` | `praxis-consumer-config` | Name of the generated `ConfigMap` in the gateway namespace. | | `clusterEndpoints[]` | `[]` | Endpoint topology for `load_balancer` clusters. Each entry maps a candidate cluster name to an address with explicit `transport` configuration. Missing transport fails closed. | | `clusterEndpoints[].transport.mode` | _(required)_ | `mutual_tls` (CA/client cert/SNI/verify), `tls` (server-authenticated TLS), or `plaintext` (no TLS, insecure/dev-only). | @@ -300,35 +305,64 @@ Praxis `ConfigMap` generation. | `tlsCertMountPath` | `/etc/praxis/tls` | Base path for mounted TLS files used when a `clusterEndpoints[]` entry uses `mutual_tls` transport. | | `listenerPort` | `8080` | HTTP port for the generated `listeners[0].address` (`0.0.0.0:{listenerPort}`). | +`GatewayRef.supportsEmptyOverlay` is a separate, default-false capability gate. +Released Praxis images that reject an empty candidate list must not receive an +empty routing revision. Upgrade and roll every consumer to an image that accepts +the versioned empty snapshot first; only then set `supportsEmptyOverlay: true`. +Until opted in, Grid retains the last distributed revision and reports +`EmptyOverlayUnsupported` in `status.overlayStatus[]` rather than claiming the +withdrawal was served. Non-empty overlay updates are unaffected. + When `enabled: true`, the `GridNetwork` controller renders a `praxis.yaml`-keyed `ConfigMap` in the gateway namespace on each reconcile. The generated config is a complete, runnable Praxis config containing: - `listeners:` — one public listener at `0.0.0.0:{listenerPort}` - `filter_chains:` — the consumer chain with: - - `intelligent_route` candidates from the routing overlay (with `credential.secretRef` for - credential-bearing candidates) - - `credential_inject` entries (one per unique credential reference) using - `file:` sources — token bytes are never written to the `ConfigMap` - - `load_balancer` entries (one per unique candidate cluster). Every referenced + - `intelligent_route` reading the versioned routing overlay with expected + network, gateway, namespace, and local-site scope. Candidates and selection + policy are not copied into startup-only YAML. + - `credential_inject` is included when current candidates have credential + references or `enableProjectedCredentials: true`. In projected mode it + starts with an empty table and resolves later references under the + projected Secret mount; missing files fail closed. Token bytes are never + written to the `ConfigMap`. + - `load_balancer` entries for every configured endpoint, including currently + inactive providers needed for restoration. Every potentially routable cluster must have a matching `clusterEndpoints[]` entry with endpoint address and explicit `transport` configuration. `transport.mode` is the security switch — not `sni` presence. Missing transport fails closed - `admin:` — admin listener at `127.0.0.1:9901` - `shutdown_timeout_secs: 5` -The generated credential-injection config assumes the gateway is the egress -component for the selected backend. This is correct for direct API-provider and -cloud-provider fallback routes. For remote provider sites, provider credentials -should be mounted only in the remote site or provider-side component that makes -the final backend call. +The generated credential-injection config is for credentials intentionally +projected into the consumer gateway's namespace. For remote provider sites, +credentials should normally be mounted only in the remote site or provider-side +component that makes the final backend call. The `credential_inject` filter is a Praxis AI runtime dependency. The AGN operator can render the config shape, but the deployed Praxis AI image must include that filter for the generated config to start successfully. -When `enabled: false` or `consumerConfig` is absent, this gateway behaves as before -— only the routing overlay `ConfigMap` is applied. +The consumer Deployment must mount the gateway's `grid-overlay--` +ConfigMap key `routing-overlay.json` at `/etc/praxis/routing/routing-overlay.json` +as a projected volume (never with `subPath`). Cross-cluster consumers must +deliver both the Praxis config ConfigMap and routing overlay to the consumer +cluster. The initial migration from static inline candidates requires a +consumer rollout; listener, endpoint/TLS, or credential-table changes continue +to require a rollout, while route-only overlay revisions hot reload. + +For generated consumer configs, first set `enableProjectedCredentials: true` +and roll out the consumer so it loads the filter and mount root. Then set +`supportsProjectedCredentials: true` to attest that rollout and mount the +referenced Secret. The Grid controller does not own or restart consumer +Deployments. Until readiness is attested, credential-bearing overlay revisions +are retained. In projected mode, the generated config uses an empty credential +table and resolves the selected Secret reference at request time, failing +closed when it cannot read it. + +When `enabled: false` or `consumerConfig` is absent, this gateway behaves as +before — only the routing overlay `ConfigMap` is applied. ## GridSite diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 6df67d36b..8ff66e8f2 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -4757,9 +4757,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", @@ -5621,9 +5621,9 @@ dependencies = [ [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" [[package]] name = "yaml_serde" @@ -5657,14 +5657,14 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", - "synstructure 0.14.0", + "syn 2.0.119", + "synstructure 0.13.2", ] [[package]] diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index 1f4eb1c4a..138992f96 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -1,7 +1,7 @@ //! Applies a grid serving config: the candidate topology and the peer pollers. use std::{ - collections::HashMap, + collections::{BTreeSet, HashMap}, path::PathBuf, sync::{ Arc, Mutex, OnceLock, PoisonError, Weak, @@ -93,6 +93,9 @@ pub(crate) struct Topology { /// Freshness window the order reads, milliseconds. load_window_ms: i64, + + /// Explicitly authenticated provider-gateway hop clusters. + provider_hop_clusters: BTreeSet, } impl Topology { @@ -104,6 +107,7 @@ impl Topology { base: Arc::from(base), local_site: Arc::from(config.local_site.as_str()), load_window_ms: config.load_window_ms, + provider_hop_clusters: validate_provider_hop_clusters(config.provider_hop_clusters.clone())?, }) } @@ -464,6 +468,15 @@ fn validate_config(config: &GridServingConfig) -> Result validate_local_site(&config.local_site)?; validate_candidates(config.candidates.clone())?; let topology = Topology::from_config(config)?; + for candidate in &config.candidates { + if topology.provider_hop_clusters.contains(&candidate.cluster) && candidate.stable_id.is_none() { + return Err(format!( + "grid: candidate '{}' on provider-hop cluster '{}' is missing stable_id", + candidate.name, candidate.cluster + ) + .into()); + } + } let mut sites = std::collections::HashSet::with_capacity(config.peers.len()); for peer in &config.peers { validate_peer(peer)?; @@ -815,6 +828,7 @@ mod tests { window_secs: 60, load_window_ms: 30_000, candidates: sites.iter().map(|site| candidate(site)).collect(), + provider_hop_clusters: Vec::new(), peers: sites.iter().map(|site| peer(site)).collect(), } } diff --git a/gateway/ai-grid-filters/src/descriptor.rs b/gateway/ai-grid-filters/src/descriptor.rs index 4fe3d33d6..94e241957 100644 --- a/gateway/ai-grid-filters/src/descriptor.rs +++ b/gateway/ai-grid-filters/src/descriptor.rs @@ -5,7 +5,10 @@ //! Data model only. Selection and ordering live in the `route` and `snapshot` //! siblings. -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{BTreeSet, HashSet}, + sync::Arc, +}; use praxis_core::connectivity::Upstream; use praxis_filter::FilterError; @@ -112,6 +115,11 @@ pub struct CandidateConfig { /// Site that owns this capability. pub site: String, + + /// Stable Grid overlay identity, used by authenticated provider-hop + /// gateways. Older/static configs may omit it and retain the derived ID. + #[serde(default)] + pub stable_id: Option, } /// Default freshness state for candidates. @@ -198,9 +206,21 @@ pub(crate) fn default_stable_id(kind: CapabilityKind, name: &str, site: &str, cl /// [`FilterError`] if the list exceeds [`MAX_CANDIDATES`], any /// name/site/cluster field is blank or oversized, or a duplicate /// (kind, name, site, cluster) tuple exists. +pub(crate) fn validate_candidates(raw: Vec) -> Result, FilterError> { + validate_candidates_with_empty(raw, false) +} + +/// Validate candidates rendered from the operator's versioned serving +/// snapshot, where an empty list is an authoritative no-route revision. +pub(crate) fn validate_serving_candidates(raw: Vec) -> Result, FilterError> { + validate_candidates_with_empty(raw, true) +} + +/// Validate candidate fields while allowing emptiness only for versioned +/// serving snapshots. #[expect( clippy::too_many_lines, - reason = "single validation loop, splitting hurts readability" + reason = "one validation pass keeps all candidate invariants adjacent" )] pub(crate) fn validate_candidates(raw: Vec) -> Result, FilterError> { if raw.len() > MAX_CANDIDATES { @@ -227,7 +247,13 @@ pub(crate) fn validate_candidates(raw: Vec) -> Result) -> Result) -> Result, FilterError> { + let mut clusters = BTreeSet::new(); + for (index, cluster) in raw.into_iter().enumerate() { + validate_name(&format!("provider_hop_clusters[{index}]"), &cluster)?; + if !clusters.insert(cluster) { + return Err("grid: duplicate provider_hop_clusters entry".into()); + } + } + Ok(clusters) +} + +/// Validate an overlay identity before it can be sent as an internal header. +fn validate_stable_id(index: usize, stable_id: &str) -> Result<(), FilterError> { + if stable_id.trim().is_empty() || stable_id.len() > MAX_NAME_LEN { + return Err(format!("grid: candidates[{index}].stable_id must be 1-{MAX_NAME_LEN} non-blank bytes").into()); + } + http::header::HeaderValue::from_str(stable_id) + .map(|_| ()) + .map_err(|error| format!("grid: candidates[{index}].stable_id is not a valid header value: {error}").into()) +} + /// Validate credential reference fields on a candidate entry. fn validate_credential(index: usize, credential: Option<&CandidateCredential>) -> Result<(), FilterError> { let Some(credential) = credential else { @@ -358,6 +406,7 @@ mod tests { kind, name: name.to_owned(), site: site.to_owned(), + stable_id: None, } } @@ -410,6 +459,19 @@ mod tests { assert!(err.to_string().contains("duplicate candidate"), "{err}"); } + #[test] + fn explicit_stable_id_is_preserved_and_header_safe() { + let mut candidate = candidate("inference_model", "llama", "site-a", "gateway-a"); + candidate.stable_id = Some("257a9450".to_owned()); + let validated = validate_candidates(vec![candidate.clone()]).expect("valid stable ID"); + assert_eq!(&*validated[0].stable_id, "257a9450"); + + candidate.stable_id = Some("\nspoof".to_owned()); + validate_candidates(vec![candidate.clone()]).expect_err("newline stable ID is rejected"); + candidate.stable_id = Some(" ".to_owned()); + validate_candidates(vec![candidate]).expect_err("blank stable ID is rejected"); + } + #[test] fn same_model_different_site_not_duplicate() { validate_candidates(vec![ diff --git a/gateway/ai-grid-filters/src/flow.rs b/gateway/ai-grid-filters/src/flow.rs index 9e5d68a2a..58ba7cafe 100644 --- a/gateway/ai-grid-filters/src/flow.rs +++ b/gateway/ai-grid-filters/src/flow.rs @@ -178,6 +178,7 @@ fn candidates(model: &str, sites: &[(&str, &str)]) -> Vec { kind: CapabilityKind::InferenceModel, name: model.to_owned(), site: (*site).to_owned(), + stable_id: None, }) .collect(); validate_candidates(raw).expect("valid candidates") diff --git a/gateway/ai-grid-filters/src/route.rs b/gateway/ai-grid-filters/src/route.rs index d22fb6d2f..267e52204 100644 --- a/gateway/ai-grid-filters/src/route.rs +++ b/gateway/ai-grid-filters/src/route.rs @@ -42,6 +42,13 @@ use crate::{ snapshot::RouteSnapshot, }; +/// Internal request header carrying the selected candidate's stable ID. +const SELECTED_CANDIDATE_HEADER: &str = "x-ai-routing-candidate"; +/// Internal request header carrying the provider-hop request correlation ID. +const PROVIDER_HOP_REQUEST_ID_HEADER: &str = "x-ai-routing-request-id"; +/// Internal request header carrying the serving overlay revision. +const OVERLAY_REVISION_HEADER: &str = "x-ai-routing-revision"; + /// Default request header carrying the model name. fn default_model_header() -> String { "X-Model".to_owned() @@ -1111,6 +1118,7 @@ mod tests { kind: CapabilityKind::InferenceModel, name: model.to_owned(), site: site.to_owned(), + stable_id: None, }]) .unwrap(); candidates[0].admission_state = admission; diff --git a/gateway/ai-grid-filters/src/serving.rs b/gateway/ai-grid-filters/src/serving.rs index 6df75b1dc..e849772bf 100644 --- a/gateway/ai-grid-filters/src/serving.rs +++ b/gateway/ai-grid-filters/src/serving.rs @@ -60,6 +60,10 @@ pub struct GridServingConfig { /// The candidate topology: which sites serve which capabilities. pub candidates: Vec, + /// Clusters allowed to carry authenticated provider-gateway hops. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub provider_hop_clusters: Vec, + /// Peers to poll for live load. pub peers: Vec, } diff --git a/gateway/ai-grid-filters/src/snapshot.rs b/gateway/ai-grid-filters/src/snapshot.rs index a721f0f55..c6192995e 100644 --- a/gateway/ai-grid-filters/src/snapshot.rs +++ b/gateway/ai-grid-filters/src/snapshot.rs @@ -91,6 +91,23 @@ impl RouteSnapshot { candidates: Vec, local_site: Arc, inputs: &mut Inputs<'_, S>, + ) -> Self { + Self::from_store_with_provider_hops(candidates, local_site, store, now_ms, window_ms, BTreeSet::new()) + } + + /// Order candidates by load and keep the provider-hop allowlist in the same + /// immutable snapshot so route and trust decisions change atomically. + #[expect( + clippy::too_many_arguments, + reason = "the immutable snapshot constructor takes all independent routing inputs explicitly" + )] + pub fn from_store_with_provider_hops( + candidates: Vec, + local_site: Arc, + store: &LoadStore, + now_ms: i64, + window_ms: i64, + provider_hop_clusters: BTreeSet, ) -> Self { // Score each candidate once, then sort the pairs: load_of allocates a // store key and scans a window, too costly to repeat inside sort_by. @@ -509,6 +526,7 @@ mod tests { kind: CapabilityKind::InferenceModel, name: name.to_owned(), site: site.to_owned(), + stable_id: None, } } diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 28ee52d29..d90c093e6 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -54,6 +54,7 @@ prometheus = { workspace = true } schemars = { workspace = true } reqwest = { workspace = true } rmcp = { workspace = true } +serde_yaml = { workspace = true } rustls = { workspace = true, optional = true } openssl = { workspace = true, optional = true } openssl-sys = { workspace = true, optional = true } @@ -79,7 +80,6 @@ zeroize = { workspace = true } rmcp = { workspace = true, features = ["server", "transport-streamable-http-server"] } pem = { workspace = true } rcgen = { workspace = true, features = ["x509-parser"] } -serde_yaml = { workspace = true } tokio = { workspace = true, features = ["test-util"] } tower = { workspace = true } diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index e4f9dd995..430c1594b 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -40,7 +40,7 @@ use crate::{ OverlayRevisionStatus, PeerTrustMode, SignalMode, SiteIdentityStatus, TenantBudgetStatus, TransportMode, }, grid_site::{GridSite, GridSitePhase, GridSiteStatus}, - inference_provider::InferenceProvider, + inference_provider::{InferenceProvider, InferenceProviderStatus}, }, error::OperatorError, readiness, @@ -2354,9 +2354,6 @@ async fn reconcile_routing_overlay_inner( } } - // Opt-in: generate and apply the consumer Praxis config when enabled. - // Render/apply errors are recorded as per-gateway status and do NOT - // abort the reconcile loop — other gateways continue to be processed. // Gateways with consumerConfig.enabled=false get a Disabled entry. // Gateways without a consumerConfig block are omitted from status. if !delegated_mount_reconciled && let Some(cc) = gw_ref.consumer_config.as_ref().filter(|cc| cc.enabled) { @@ -2487,6 +2484,12 @@ fn render_serving_text( .map_or(crate::crd::grid_network::DEFAULT_TLS_CERT_MOUNT_PATH, |cc| { cc.tls_cert_mount_path.as_str() }); + let provider_hop_clusters = gw_ref + .consumer_config + .as_ref() + .map(|config| consumer_config::provider_hop_clusters(&config.cluster_endpoints)) + .transpose()? + .unwrap_or_default(); let inputs = ServingInputs { tls_mount, local_signals_addr: source.settings.local_signals_addr.as_deref(), @@ -2517,7 +2520,12 @@ async fn apply_serving_config( .map(String::as_str); let key = format!("{}/{name}", gw_ref.namespace); let now = Instant::now(); - match serving_config::decide_write(stored, &text, source.gate.last(&key), now) { + let decision = if overlay.candidates.is_empty() { + serving_config::decide_empty_withdrawal(stored, &text) + } else { + serving_config::decide_write(stored, &text, source.gate.last(&key), now) + }; + match decision { WriteDecision::Unchanged => return Ok(None), WriteDecision::Deferred(wait) => return Ok(Some(wait)), WriteDecision::Write => {}, @@ -4298,11 +4306,19 @@ fn determine_phase(network: &GridNetwork, grid_id: &str, membership: Option<&Mem /// All variants are preserved so remote sites know about unavailable providers /// and can avoid routing to them. Absent status (not yet reconciled) maps to /// `Pending`. -fn crdt_phase_from_provider( - status_phase: Option<&crate::crd::inference_provider::ProviderPhase>, -) -> crdt::ProviderPhase { +fn crdt_phase_from_provider(status: Option<&InferenceProviderStatus>, generation: i64) -> crdt::ProviderPhase { use crate::crd::inference_provider::ProviderPhase as Op; - match status_phase { + if status.is_some_and(|status| { + status.phase == Op::Pending + && status.reason.as_deref() == Some("NoMatchingSites") + && status.observed_generation == generation + }) { + // The provider controller has observed this generation and confirmed + // that its selector matches no known site. Preserve that withdrawal in + // the CRDT instead of publishing Pending as routable capacity. + return crdt::ProviderPhase::Unavailable; + } + match status.map(|status| &status.phase) { Some(Op::Available) => crdt::ProviderPhase::Available, Some(Op::Degraded) => crdt::ProviderPhase::Degraded, Some(Op::Unavailable) => crdt::ProviderPhase::Unavailable, @@ -4375,6 +4391,10 @@ fn log_capacity_changes(logged: &ChangeLog, network: &str, providers: &[Inferenc /// advances on spec and status writes, and falls back to `metadata.generation` /// when no parseable resource version is present. Equal revisions break ties /// via `writer_id`, which is the advertising SWIM site identity. +#[expect( + clippy::too_many_lines, + reason = "the provider-to-CRDT mapping keeps every advertised field derived from one source object" +)] fn provider_state_from_kube( provider: &InferenceProvider, network_id: &str, @@ -6867,28 +6887,60 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "this table proves current and stale withdrawal behavior for every provider phase" + )] fn crdt_phase_from_provider_maps_all_variants() { - use crate::crd::inference_provider::ProviderPhase as Op; - assert_eq!( - crdt_phase_from_provider(None), + crdt_phase_from_provider(None, 1), crdt::ProviderPhase::Pending, "absent status → Pending" ); + let pending: InferenceProviderStatus = serde_json::from_value(serde_json::json!({ + "phase": "Pending", "observedGeneration": 3 + })) + .unwrap_or_else(|_| std::process::abort()); assert_eq!( - crdt_phase_from_provider(Some(&Op::Pending)), + crdt_phase_from_provider(Some(&pending), 3), crdt::ProviderPhase::Pending ); + let withdrawn: InferenceProviderStatus = serde_json::from_value(serde_json::json!({ + "phase": "Pending", "reason": "NoMatchingSites", "matchingSites": [], "observedGeneration": 3 + })) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!( + crdt_phase_from_provider(Some(&withdrawn), 3), + crdt::ProviderPhase::Unavailable, + "current-generation selector withdrawal must not be advertised as routable capacity" + ); + assert_eq!( + crdt_phase_from_provider(Some(&withdrawn), 4), + crdt::ProviderPhase::Pending, + "stale withdrawal status must not withdraw a newer provider generation" + ); + let available: InferenceProviderStatus = serde_json::from_value(serde_json::json!({ + "phase": "Available", "observedGeneration": 3 + })) + .unwrap_or_else(|_| std::process::abort()); assert_eq!( - crdt_phase_from_provider(Some(&Op::Available)), + crdt_phase_from_provider(Some(&available), 3), crdt::ProviderPhase::Available ); + let degraded: InferenceProviderStatus = serde_json::from_value(serde_json::json!({ + "phase": "Degraded", "observedGeneration": 3 + })) + .unwrap_or_else(|_| std::process::abort()); assert_eq!( - crdt_phase_from_provider(Some(&Op::Degraded)), + crdt_phase_from_provider(Some(°raded), 3), crdt::ProviderPhase::Degraded ); + let unavailable: InferenceProviderStatus = serde_json::from_value(serde_json::json!({ + "phase": "Unavailable", "observedGeneration": 3 + })) + .unwrap_or_else(|_| std::process::abort()); assert_eq!( - crdt_phase_from_provider(Some(&Op::Unavailable)), + crdt_phase_from_provider(Some(&unavailable), 3), crdt::ProviderPhase::Unavailable ); } @@ -8693,6 +8745,7 @@ mod tests { name: name.to_owned(), namespace: ns.to_owned(), local_site_name: None, + supports_empty_overlay: false, consumer_config: None, } } @@ -8819,7 +8872,7 @@ mod tests { } #[test] - fn retained_overlay_status_preserves_last_distributed_revision() { + fn failed_overlay_status_preserves_last_distributed_revision() { let gw = make_gw_ref("gw", "grid-system"); let prior = rendered_overlay_status(&gw); let mut network = base_network(); @@ -8828,7 +8881,7 @@ mod tests { ..GridNetworkStatus::default() }); - let status = retained_overlay_status(&network, &gw, 5, None, "EmptyCandidates", "no candidates available"); + let status = retained_overlay_status(&network, &gw, 5, None, "OverlayApplyFailed", "overlay apply failed"); assert_eq!(status.phase, OverlayPhase::Retained); assert_eq!(status.rendered_revision, prior.rendered_revision); @@ -8837,7 +8890,7 @@ mod tests { assert_eq!(status.config_map_resource_version, prior.config_map_resource_version); assert_eq!(status.candidate_count, prior.candidate_count); assert_eq!(status.rendered_at, prior.rendered_at); - assert_eq!(status.reason, "EmptyCandidates"); + assert_eq!(status.reason, "OverlayApplyFailed"); assert!(status.message.contains("previous valid overlay retained")); assert_eq!(status.observed_generation, 5); } @@ -8890,12 +8943,12 @@ mod tests { ..GridNetworkStatus::default() }); - let status = retained_overlay_status(&network, &gw, 2, None, "EmptyCandidates", "no candidates available"); + let status = retained_overlay_status(&network, &gw, 2, None, "OverlayRenderFailed", "overlay render failed"); assert_eq!(status.phase, OverlayPhase::Error); assert!(status.rendered_revision.is_empty()); assert!(status.distributed_revision.is_empty()); - assert_eq!(status.reason, "EmptyCandidates"); + assert_eq!(status.reason, "OverlayRenderFailed"); } // ----------------------------------------------------------------------- @@ -8940,41 +8993,6 @@ mod tests { assert!(!grid_network_status_needs_update(network.status.as_ref(), &desired)); } - #[test] - fn unchanged_empty_candidates_overlay_writes_no_status() { - let gw = make_gw_ref("gw", "grid-system"); - let mut network = network_with_overlay(rendered_overlay_status(&gw)); - let first_render = make_render_result(&"c".repeat(64), 0); - let first = retained_overlay_status(&network, &gw, 4, Some(&first_render), EMPTY_CANDIDATES, "no candidates"); - let first = desired_with_overlay(&network, first); - network.status = Some(first.clone()); - let mut next_render = make_render_result(&"c".repeat(64), 0); - FRESH.clone_into(&mut next_render.rendered_at); - - let next = retained_overlay_status(&network, &gw, 4, Some(&next_render), EMPTY_CANDIDATES, "no candidates"); - let desired = desired_with_overlay(&network, next); - - assert_eq!(only_overlay(&desired).rendered_at, only_overlay(&first).rendered_at); - assert!(!grid_network_status_needs_update(network.status.as_ref(), &desired)); - } - - #[test] - fn unchanged_overlay_without_prior_distribution_writes_no_status() { - let gw = make_gw_ref("gw", "grid-system"); - let mut network = base_network(); - let first_render = make_render_result(&"c".repeat(64), 0); - let first = retained_overlay_status(&network, &gw, 1, Some(&first_render), EMPTY_CANDIDATES, "no candidates"); - network.status = Some(desired_with_overlay(&network, first)); - let mut next_render = make_render_result(&"c".repeat(64), 0); - FRESH.clone_into(&mut next_render.rendered_at); - - let next = retained_overlay_status(&network, &gw, 1, Some(&next_render), EMPTY_CANDIDATES, "no candidates"); - let desired = desired_with_overlay(&network, next); - - assert_eq!(only_overlay(&desired).phase, OverlayPhase::Error); - assert!(!grid_network_status_needs_update(network.status.as_ref(), &desired)); - } - #[test] fn content_change_advances_rendered_at() { let gw = make_gw_ref("gw", "grid-system"); @@ -8993,9 +9011,9 @@ mod tests { ..prior.clone() }; let new_reason = OverlayRevisionStatus { - phase: OverlayPhase::Retained, - reason: EMPTY_CANDIDATES.to_owned(), - message: "no candidates".to_owned(), + phase: OverlayPhase::Error, + reason: "OverlayApplyFailed".to_owned(), + message: "apply failed".to_owned(), rendered_at: FRESH.to_owned(), ..prior }; @@ -9136,25 +9154,6 @@ mod tests { assert_eq!(status.phase, OverlayPhase::Error); } - #[test] - fn retained_status_empty_candidates_retains_prior_distribution() { - let gw = make_gw_ref("gw", "grid-system"); - let prior = rendered_overlay_status(&gw); - let mut network = base_network(); - network.status = Some(GridNetworkStatus { - overlay_status: vec![prior.clone()], - ..GridNetworkStatus::default() - }); - let render = make_render_result(&"c".repeat(64), 0); - - let status = retained_overlay_status(&network, &gw, 6, Some(&render), "EmptyCandidates", "no candidates"); - - assert_eq!(status.rendered_revision, "c".repeat(64)); - assert_eq!(status.distributed_revision, prior.distributed_revision); - assert_eq!(status.candidate_count, 0); - assert_eq!(status.phase, OverlayPhase::Retained); - } - #[test] fn retained_status_render_failure_preserves_all_prior_evidence() { let gw = make_gw_ref("gw", "grid-system"); @@ -9795,6 +9794,16 @@ mod tests { ); } + #[test] + fn consumer_config_status_projected_credentials_reason_is_specific() { + let gw = make_gw_ref("gw", "ns"); + let cc = make_consumer_config("cm"); + let err = OperatorError::ConsumerConfigRender(ConsumerConfigError::ProjectedCredentialsUnsupported); + let status = consumer_config_status_error(&gw, &cc, &err, 1); + assert_eq!(status.reason, "ProjectedCredentialsUnsupported"); + assert!(status.message.contains("supportsProjectedCredentials=true")); + } + #[test] fn consumer_config_status_plaintext_with_sni_reason() { let gw = make_gw_ref("gw", "ns"); diff --git a/operator/src/controller/inference_provider.rs b/operator/src/controller/inference_provider.rs index b42af45f2..15bb3aad3 100644 --- a/operator/src/controller/inference_provider.rs +++ b/operator/src/controller/inference_provider.rs @@ -281,6 +281,20 @@ pub fn phase_from_probe(outcome: ProbeOutcome, site_phase: ProviderPhase) -> Pro /// This function never returns [`ProviderPhase::Degraded`]. /// `Degraded` is only reachable via [`phase_from_probe`] when a health /// probe returns a degraded response. +/// Mark a current provider generation as withdrawn only when known site +/// inventory exists and the provider selector matches none of it. With no +/// inventory, the existing Phase 1 fallback remains eligible. +fn selector_has_no_matches(site_count: usize, matching: &[String]) -> bool { + site_count > 0 && matching.is_empty() +} + +/// Status reason that lets remote Grid sites distinguish intentional selector +/// withdrawal from an uninitialized provider (`Pending`) or absent inventory. +fn no_matching_sites_reason(has_no_matches: bool) -> Option { + has_no_matches.then(|| "NoMatchingSites".to_owned()) +} + +/// Derive the initial provider phase from the current selector matches. pub(crate) fn phase_from_matching(matching: &[String]) -> ProviderPhase { if matching.is_empty() { ProviderPhase::Pending @@ -644,8 +658,17 @@ async fn resolve_phase_and_sites( // Resolve matching sites. let sites = list_sites_for_network(client, network_ref).await?; let matching = hosting_sites(provider, &sites, local_site); + let selector_has_no_matches = selector_has_no_matches(sites.len(), &matching); let site_phase = phase_from_matching(&matching); + // A known-empty selector result is an authoritative withdrawal. Resolve it + // before health/TLS probes: an unrelated probe failure must not turn this + // provider back into Degraded, which remains routable as stale capacity in + // the cross-site CRDT. + if selector_has_no_matches { + return Ok((ProviderPhase::Pending, matching, no_matching_sites_reason(true))); + } + // Resolve health check TLS config (if configured). On failure, map // the error to a structured status reason and mark the provider Degraded. let health_tls_config = if let Some(hc) = &provider.spec.health_check @@ -689,6 +712,7 @@ async fn resolve_phase_and_sites( Err(HealthProbeError::Operator(error)) => return Err(error), }; let phase = phase_from_probe(probe_result, site_phase); + let reason = no_matching_sites_reason(selector_has_no_matches); // Validate metrics TLS configuration. if phase == ProviderPhase::Available @@ -704,7 +728,7 @@ async fn resolve_phase_and_sites( return Ok((ProviderPhase::Degraded, matching, Some(tls_reason))); } - Ok((phase, matching, None)) + Ok((phase, matching, reason)) } /// List all [`GridSite`]s whose `spec.gridNetworkRef` matches `network_ref`. @@ -1610,6 +1634,25 @@ mod tests { assert_eq!(phase, ProviderPhase::Pending, "empty matching → Pending"); } + #[test] + fn no_matching_sites_reason_takes_precedence_over_probe_phase() { + assert!(selector_has_no_matches(1, &[])); + assert!( + !selector_has_no_matches(0, &[]), + "no inventory preserves Phase 1 fallback" + ); + assert!(!selector_has_no_matches(1, &["site-a".to_owned()])); + // The reason is based on authoritative selector inventory, not the + // eventual health-probe phase. resolve_phase_and_sites returns this + // Pending withdrawal before it can become Degraded. + assert_eq!(no_matching_sites_reason(true).as_deref(), Some("NoMatchingSites")); + assert_eq!(no_matching_sites_reason(false), None); + assert_eq!( + phase_from_probe(ProbeOutcome::Degraded, ProviderPhase::Pending), + ProviderPhase::Degraded + ); + } + #[test] fn one_matching_site_yields_available() { // Item 7: valid config, ≥1 matching site → Available diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index b00793be9..f4e88db1a 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -746,6 +746,14 @@ pub struct GatewayRef { #[serde(default)] pub local_site_name: Option, + /// Explicitly attest that every consumer of this gateway's routing overlay + /// accepts a valid versioned snapshot with `candidates: []` as a no-route + /// decision. Defaults to `false` for compatibility with released Praxis + /// images that reject empty candidate lists. The operator retains the last + /// distributed revision until this is enabled. + #[serde(default)] + pub supports_empty_overlay: bool, + /// Opt-in configuration for operator-managed consumer Praxis config generation. /// /// When absent or `enabled: false`, this gateway behaves exactly as before — @@ -757,7 +765,9 @@ pub struct GatewayRef { /// entry per unique inference cluster. Other capability kinds remain in the /// routing overlay for dedicated data-plane pipelines. /// - /// The generated `ConfigMap` contains no token bytes. + /// The generated `ConfigMap` contains no token bytes. Credential-bearing + /// overlays are withheld until the consumer has explicitly declared that its + /// compatible filter config is already running. #[serde(default, skip_serializing_if = "Option::is_none")] pub consumer_config: Option, } @@ -790,10 +800,32 @@ pub struct ConsumerConfig { #[serde(default)] pub enabled: bool, + /// Generate the dynamic `credential_inject` filter and projected mount + /// root, including when the current overlay has no credentials. Enable this + /// first, then roll out the generated config before asserting readiness + /// with `supportsProjectedCredentials`. + /// + /// Default: `false`. + #[serde(default)] + pub enable_projected_credentials: bool, + + /// Declare that the running consumer has the dynamic `credential_inject` + /// filter configured with the projected Secret mount root. This is a + /// fail-closed capability gate for credential-bearing overlay revisions. + /// Set it only after the consumer has loaded a config containing that + /// filter; the Grid operator does not own or roll out the consumer + /// Deployment. + /// + /// Default: `false`. + #[serde(default)] + pub supports_projected_credentials: bool, + /// Base directory for mounted credential Secret files inside the consumer pod. /// - /// Each credential Secret is expected to be mounted at - /// `{credentialMountBase}/{secret-name}/{secret-key}`. + /// In projected-credential mode, mount each Secret at + /// `{credentialMountBase}/{secret-namespace}/{secret-name}` with its data + /// keys as files. Static `file:` entries continue to use their explicit + /// paths and are not constrained by this directory layout. /// /// Default: `/run/secrets/grid-credentials`. #[serde(default = "default_credential_mount_base")] @@ -987,6 +1019,8 @@ impl Default for ConsumerConfig { fn default() -> Self { Self { enabled: false, + enable_projected_credentials: false, + supports_projected_credentials: false, credential_mount_base: default_credential_mount_base(), config_map_name: default_consumer_config_map_name(), cluster_endpoints: Vec::new(), @@ -1729,6 +1763,31 @@ mod tests { gateway_ref_properties.contains_key("localSiteName"), "CRD schema must include localSiteName field on GatewayRef" ); + assert!( + gateway_ref_properties.contains_key("supportsEmptyOverlay"), + "CRD schema must include the empty-overlay capability gate" + ); + } + + #[test] + fn gateway_ref_empty_overlay_capability_defaults_closed_and_round_trips() { + let mut gw: GatewayRef = serde_json::from_value(serde_json::json!({ + "name": "gw", + "namespace": "ns" + })) + .unwrap_or_else(|_| std::process::abort()); + assert!( + !gw.supports_empty_overlay, + "old GatewayRef documents default to unsupported" + ); + gw.supports_empty_overlay = true; + let serialized = serde_json::to_value(gw).unwrap_or_else(|_| std::process::abort()); + assert_eq!( + serialized + .get("supportsEmptyOverlay") + .and_then(serde_json::Value::as_bool), + Some(true) + ); } // ----------------------------------------------------------------------- @@ -1753,6 +1812,8 @@ mod tests { "namespace": "ns", "consumerConfig": { "enabled": true, + "enableProjectedCredentials": true, + "supportsProjectedCredentials": true, "credentialMountBase": "/run/secrets/grid", "configMapName": "my-consumer-config", "tlsCertMountPath": "/etc/custom-tls", @@ -1774,6 +1835,14 @@ mod tests { let gw: GatewayRef = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); let cc = gw.consumer_config.unwrap_or_else(|| std::process::abort()); assert!(cc.enabled, "enabled must round-trip"); + assert!( + cc.enable_projected_credentials, + "projected filter opt-in must round-trip" + ); + assert!( + cc.supports_projected_credentials, + "projected credential readiness must round-trip" + ); assert_eq!( cc.credential_mount_base, "/run/secrets/grid", "credentialMountBase must round-trip" @@ -1842,6 +1911,10 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "asserts every ConsumerConfig default as one contract" + )] fn consumer_config_defaults_when_subfields_absent() { let json = serde_json::json!({ "name": "gw", @@ -1851,6 +1924,14 @@ mod tests { let gw: GatewayRef = serde_json::from_value(json).unwrap_or_else(|_| std::process::abort()); let cc = gw.consumer_config.unwrap_or_else(|| std::process::abort()); assert!(!cc.enabled, "enabled must default to false"); + assert!( + !cc.enable_projected_credentials, + "projected filter opt-in must default off" + ); + assert!( + !cc.supports_projected_credentials, + "projected credentials must default closed" + ); assert_eq!( cc.credential_mount_base, "/run/secrets/grid-credentials", "credentialMountBase must use default" @@ -1985,6 +2066,7 @@ mod tests { name: "gw".to_owned(), namespace: "ns".to_owned(), local_site_name: None, + supports_empty_overlay: false, consumer_config: None, }; let json = serde_json::to_value(&gw).unwrap_or_else(|_| std::process::abort()); diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index 2e38e7f0e..ee69440cd 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -32,6 +32,9 @@ use crate::{ resources::routing_overlay::{RoutingCandidate, RoutingOverlay}, }; +/// Path at which a consumer gateway must mount the operator-published overlay. +pub(crate) const CONSUMER_OVERLAY_FILE: &str = "/etc/praxis/routing/routing-overlay.json"; + // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- @@ -342,6 +345,7 @@ pub(crate) fn generate_consumer_praxis_config_with_telemetry( let candidates_yaml = render_candidates(&inference_candidates, &overlay.local_site); let selection_policy_yaml = render_selection_policy(overlay.selection_policy.as_ref()); + let provider_hop_clusters_yaml = render_provider_hop_clusters(cluster_endpoints)?; let local_site = yaml_scalar(&overlay.local_site)?; let trace_context_filter = if telemetry.is_some() { " - filter: trace_context\n" @@ -370,14 +374,13 @@ pub(crate) fn generate_consumer_praxis_config_with_telemetry( \x20 - filter: intelligent_route\n\ \x20 local_site: {local_site}\n\ \x20 model_header: \"X-Model\"\n\ + {provider_hop_clusters_yaml}\ {selection_policy_yaml}\ \x20 candidates:\n\ {candidates_yaml}" ); - if let Some(inject) = credential_inject_section { - config.push_str(&inject); - } + config.push_str(&credential_inject_section); config.push_str(&load_balancer_section); @@ -792,6 +795,73 @@ fn render_selection_policy(policy: Option<&crate::crd::grid_network::SelectionPo format!(" selection_policy:\n mode: {mode}\n") } +/// Render provider-hop context-header configuration for mTLS endpoints. +/// +/// `clusterEndpoints` describes consumer-to-provider-gateway endpoints. An +/// explicit mTLS transport identifies the authenticated provider-gateway hop; +/// those cluster names need the routing-context headers consumed by the +/// provider's `provider_route` filter. Plaintext endpoints remain direct/local +/// backends and do not receive provider-hop headers. +fn render_provider_hop_clusters(cluster_endpoints: &[ClusterEndpointConfig]) -> Result { + let clusters = provider_hop_clusters(cluster_endpoints)?; + if clusters.is_empty() { + return Ok(String::new()); + } + let values = clusters + .into_iter() + .map(|cluster| yaml_scalar(&cluster).unwrap_or_else(|_| "\"\"".to_owned())) + .collect::>() + .join(", "); + Ok(format!(" provider_hop_clusters: [{values}]\n")) +} + +/// The explicit mTLS endpoint names allowed to receive provider-hop context. +/// Validate the endpoint identities here because this allowlist is also used by +/// the embedded Grid gateway, independently of generated Praxis config. +#[expect( + clippy::too_many_lines, + reason = "this validation keeps the mTLS provider-hop boundary explicit" +)] +pub(crate) fn provider_hop_clusters( + cluster_endpoints: &[ClusterEndpointConfig], +) -> Result, ConsumerConfigError> { + let mut seen = BTreeSet::new(); + let mut hops = BTreeSet::new(); + for endpoint in cluster_endpoints { + if endpoint.cluster.trim().is_empty() { + return Err(ConsumerConfigError::BlankCluster { + kind: "cluster_endpoint".to_owned(), + name: endpoint.cluster.clone(), + }); + } + if !seen.insert(endpoint.cluster.as_str()) { + return Err(ConsumerConfigError::DuplicateClusterEndpoint { + cluster: endpoint.cluster.clone(), + }); + } + match endpoint.transport.as_ref() { + Some(transport) if transport.mode == TransportMode::MutualTls => { + if transport.sni.as_deref().is_none_or(|sni| sni.trim().is_empty()) { + return Err(ConsumerConfigError::MissingSni { + cluster: endpoint.cluster.clone(), + }); + } + hops.insert(endpoint.cluster.clone()); + }, + Some(transport) + if transport.mode == TransportMode::Plaintext + && transport.sni.as_deref().is_some_and(|sni| !sni.trim().is_empty()) => + { + return Err(ConsumerConfigError::PlaintextWithSni { + cluster: endpoint.cluster.clone(), + }); + }, + Some(_) | None => {}, + } + } + Ok(hops) +} + /// Render one `intelligent_route` candidate. #[expect( clippy::too_many_lines, @@ -859,11 +929,10 @@ fn render_credential_reference(cred: &crate::resources::routing_overlay::Project ] } -/// Render the `credential_inject` filter section. -/// -/// Returns `None` when no credential-bearing candidates exist. -/// Each unique `(strategy, name, namespace, key)` tuple produces one entry. -/// Entries use `file:` sources; no `value:` is ever emitted. +/// Render `credential_inject` for current credential-bearing candidates, or +/// unconditionally in explicit projected-credential mode. The latter keeps a +/// filter present at empty-overlay startup so a later credential-bearing +/// revision cannot bypass injection. Missing projected files fail closed. #[expect( clippy::too_many_lines, reason = "BTreeMap collection + format strings for each credential field" @@ -910,17 +979,31 @@ fn render_credential_inject( entries.insert(map_key, entry); } - if entries.is_empty() { - return None; + if entries.is_empty() && !enable_projected_credentials { + return String::new(); } - - Some(format!( + let credentials = if entries.is_empty() { + " credentials: []\n".to_owned() + } else { + format!( + " credentials:\n{}\n", + entries.into_values().collect::>().join("\n") + ) + }; + let projected_base = if enable_projected_credentials { + format!( + " projected_credential_mount_base: {}\n", + yaml_scalar(credential_mount_base).unwrap_or_else(|_| "\"\"".to_owned()) + ) + } else { + String::new() + }; + format!( "\n\ \x20 - filter: credential_inject\n\ - \x20 credentials:\n\ - {}", - entries.into_values().collect::>().join("\n") - )) + {credentials}\ + {projected_base}" + ) } /// Render the `load_balancer` filter section. @@ -1554,6 +1637,215 @@ mod tests { .collect() } + #[test] + #[expect( + clippy::too_many_lines, + reason = "asserts the generated filter chain and complete restoration endpoint inventory" + )] + fn production_consumer_config_uses_scoped_reloadable_overlay_and_complete_endpoint_inventory() { + let overlay = simple_overlay(vec![credential_candidate( + "inference_model", + "model-a", + "site-a", + "provider-a", + "provider-a-secret", + "grid-system", + "token", + )]); + let mut endpoints = endpoint_coverage(&overlay); + endpoints[0].transport = Some(EndpointTransport { + mode: TransportMode::MutualTls, + sni: Some("provider-a.grid.internal".to_owned()), + }); + endpoints.push(ClusterEndpointConfig { + cluster: "provider-b".to_owned(), + address: "127.0.0.1:30002".to_owned(), + transport: Some(EndpointTransport { + mode: TransportMode::Plaintext, + sni: None, + }), + }); + + let yaml = generate_consumer_praxis_config_for_gateway( + &overlay, + MOUNT_BASE, + &endpoints, + "/etc/praxis/tls", + 8080, + "consumer-gateway", + "consumer-ns", + false, + ) + .expect("dynamic consumer config renders"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let filters = config["filter_chains"][0]["filters"].as_sequence().expect("filters"); + let route = filters + .iter() + .find(|filter| filter["filter"] == "intelligent_route") + .expect("route filter"); + assert_eq!(route["overlay_file"], CONSUMER_OVERLAY_FILE); + assert_eq!(route["expected_overlay_scope"]["network"], "test-net"); + assert_eq!(route["expected_overlay_scope"]["gateway"], "consumer-gateway"); + assert_eq!(route["expected_overlay_scope"]["namespace"], "consumer-ns"); + assert_eq!(route["expected_overlay_scope"]["local_site"], "site-a"); + assert_eq!(route["reload"]["enabled"], true); + assert_eq!(route["provider_hop_clusters"][0], "provider-a"); + assert!(route.get("candidates").is_none(), "candidate state is overlay-owned"); + assert!( + route.get("selection_policy").is_none(), + "selection mode is overlay-owned" + ); + + let load_balancer = filters + .iter() + .find(|filter| filter["filter"] == "load_balancer") + .expect("load balancer"); + let clusters = load_balancer["clusters"].as_sequence().expect("clusters"); + assert_eq!(clusters.len(), 2, "inactive endpoint remains available for restoration"); + assert!(clusters.iter().any(|cluster| cluster["name"] == "provider-b")); + assert!( + filters.iter().any(|filter| filter["filter"] == "credential_inject"), + "the active overlay credential remains configured" + ); + assert!(!yaml.contains(SENTINEL_TOKEN)); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "checks the cold-start empty overlay contract with a configured endpoint" + )] + fn empty_uncredentialed_consumer_config_does_not_require_new_praxis_filter() { + let overlay = simple_overlay(Vec::new()); + let endpoints = [ClusterEndpointConfig { + cluster: "provider-a".to_owned(), + address: "127.0.0.1:30001".to_owned(), + transport: Some(EndpointTransport { + mode: TransportMode::Plaintext, + sni: None, + }), + }]; + let yaml = generate_consumer_praxis_config_for_gateway( + &overlay, + MOUNT_BASE, + &endpoints, + "/etc/praxis/tls", + 8080, + "consumer-gateway", + "consumer-ns", + false, + ) + .expect("valid empty overlay config"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let filters = config["filter_chains"][0]["filters"].as_sequence().expect("filters"); + let route = filters + .iter() + .find(|filter| filter["filter"] == "intelligent_route") + .expect("route filter"); + assert!(route.get("candidates").is_none()); + assert_eq!(route["overlay_file"], CONSUMER_OVERLAY_FILE); + assert!( + filters.iter().all(|filter| filter["filter"] != "credential_inject"), + "old consumer images remain compatible when projected credentials are not opted in" + ); + assert_eq!( + filters + .iter() + .filter(|filter| filter["filter"] == "load_balancer") + .count(), + 1 + ); + } + + #[test] + fn projected_credential_opt_in_keeps_inject_filter_for_empty_startup_overlay() { + let overlay = simple_overlay(Vec::new()); + let endpoints = [ClusterEndpointConfig { + cluster: "provider-a".to_owned(), + address: "127.0.0.1:30001".to_owned(), + transport: Some(EndpointTransport { + mode: TransportMode::Plaintext, + sni: None, + }), + }]; + let yaml = generate_consumer_praxis_config_for_gateway( + &overlay, + MOUNT_BASE, + &endpoints, + "/etc/praxis/tls", + 8080, + "consumer-gateway", + "consumer-ns", + true, + ) + .expect("compatible consumer config renders"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let filters = config["filter_chains"][0]["filters"].as_sequence().expect("filters"); + let inject = filters + .iter() + .find(|filter| filter["filter"] == "credential_inject") + .expect("projected credential capability installs filter at cold start"); + assert_eq!(inject["credentials"], serde_yaml::Value::Sequence(Vec::new())); + assert_eq!(inject["projected_credential_mount_base"], MOUNT_BASE); + } + + #[test] + #[expect( + clippy::too_many_lines, + reason = "checks that projected credential references stay dynamic in rendered YAML" + )] + fn projected_credential_mode_keeps_reference_dynamic_for_credential_route() { + let overlay = simple_overlay(vec![credential_candidate( + "inference_model", + "model-with-secret", + "site-a", + "provider-a", + "provider-secret", + "consumer-ns", + "token", + )]); + let yaml = generate_consumer_praxis_config_for_gateway( + &overlay, + MOUNT_BASE, + &endpoint_coverage(&overlay), + "/etc/praxis/tls", + 8080, + "consumer-gateway", + "consumer-ns", + true, + ) + .expect("projected consumer config renders"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let filters = config["filter_chains"][0]["filters"].as_sequence().expect("filters"); + let inject = filters + .iter() + .find(|filter| filter["filter"] == "credential_inject") + .expect("dynamic filter is present"); + assert_eq!(inject["credentials"], serde_yaml::Value::Sequence(Vec::new())); + assert_eq!(inject["projected_credential_mount_base"], MOUNT_BASE); + assert!( + !yaml.contains("provider-secret"), + "reference identity remains only in the dynamic overlay" + ); + } + + #[test] + fn dynamic_consumer_config_requires_endpoint_inventory() { + let overlay = simple_overlay(Vec::new()); + let error = generate_consumer_praxis_config_for_gateway( + &overlay, + MOUNT_BASE, + &[], + "/etc/praxis/tls", + 8080, + "consumer-gateway", + "consumer-ns", + false, + ) + .expect_err("restoration needs at least one configured endpoint"); + assert!(matches!(error, ConsumerConfigError::NoClusterEndpoints)); + } + fn selection_policy_mode(config: &str) -> Option { let parsed: serde_yaml::Value = serde_yaml::from_str(config).ok()?; parsed @@ -1901,7 +2193,7 @@ mod tests { } #[test] - fn no_credential_candidates_produces_no_credential_inject() { + fn uncredentialed_static_config_does_not_emit_unneeded_inject_filter() { let overlay = simple_overlay(vec![plain_candidate( "inference_model", "model-x", @@ -1917,10 +2209,11 @@ mod tests { 8080, ) .unwrap(); - assert!( - !yaml.contains("credential_inject"), - "no credential candidates must produce no credential_inject" - ); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated Praxis YAML parses"); + let filters = config["filter_chains"][0]["filters"] + .as_sequence() + .expect("filter chain"); + assert!(filters.iter().all(|filter| filter["filter"] != "credential_inject")); } #[test] @@ -2538,6 +2831,64 @@ mod tests { "key path must appear" ); assert!(yaml.contains("verify: true"), "verify flag must appear"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let route = config["filter_chains"][0]["filters"] + .as_sequence() + .expect("filters") + .iter() + .find(|filter| filter["filter"] == "intelligent_route") + .expect("intelligent route"); + assert_eq!(route["provider_hop_clusters"][0], "site-a"); + } + + #[test] + fn provider_hop_clusters_include_only_explicit_mtls_endpoints() { + let overlay = simple_overlay(vec![ + plain_candidate("inference_model", "model-a", "site-a", "remote-a", true), + plain_candidate("inference_model", "model-b", "site-b", "local-b", true), + ]); + let endpoints = [ + mtls_ep("remote-a", "provider-a.example:8443", "provider-a.example"), + plain_ep("local-b", "local-backend.default.svc:8080"), + ]; + let yaml = generate_consumer_praxis_config(&overlay, MOUNT_BASE, &endpoints, "/etc/praxis/tls", 8080) + .expect("mixed-transport consumer config renders"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let route = config["filter_chains"][0]["filters"] + .as_sequence() + .expect("filters") + .iter() + .find(|filter| filter["filter"] == "intelligent_route") + .expect("intelligent route"); + assert_eq!(route["provider_hop_clusters"].as_sequence().expect("hop list").len(), 1); + assert_eq!(route["provider_hop_clusters"][0], "remote-a"); + } + + #[test] + fn plaintext_only_consumer_has_no_provider_hop_cluster_list() { + let overlay = simple_overlay(vec![plain_candidate( + "inference_model", + "model-a", + "site-a", + "local-a", + true, + )]); + let yaml = generate_consumer_praxis_config( + &overlay, + MOUNT_BASE, + &[plain_ep("local-a", "backend.default.svc:8080")], + "/etc/praxis/tls", + 8080, + ) + .expect("plaintext consumer config renders"); + let config: serde_yaml::Value = serde_yaml::from_str(&yaml).expect("generated YAML parses"); + let route = config["filter_chains"][0]["filters"] + .as_sequence() + .expect("filters") + .iter() + .find(|filter| filter["filter"] == "intelligent_route") + .expect("intelligent route"); + assert!(route.get("provider_hop_clusters").is_none()); } #[test] diff --git a/operator/src/resources/routing_overlay.rs b/operator/src/resources/routing_overlay.rs index c5ab7bae2..dd76f498b 100644 --- a/operator/src/resources/routing_overlay.rs +++ b/operator/src/resources/routing_overlay.rs @@ -3156,11 +3156,8 @@ mod tests { #[test] fn all_providers_unavailable_produces_empty_overlay() { - // If every provider in the network is Unavailable, the renderer produces - // an empty candidate list without returning an error. The reconcile-loop - // guard (in grid_network controller) skips applying an empty overlay to - // prevent Praxis hot-reload errors — that guard is covered at the - // controller integration level. This test covers the renderer contract. + // If every provider is Unavailable, the renderer emits the authoritative + // empty candidate list that the versioned overlay contract publishes. let network = test_network("empty-net"); let p1 = test_provider_with_phase("prov-a", "empty-net", &["model-a"], "Unavailable"); let p2 = test_provider_with_phase("prov-b", "empty-net", &["model-b"], "Unavailable"); diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index ec6580da6..36174e278 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -73,6 +73,9 @@ pub(crate) struct ServingConfig { pub(crate) load_window_ms: i64, /// Local site first, then by site, name, cluster. pub(crate) candidates: Vec, + /// Explicit mTLS provider gateways authorized to receive hop context. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub(crate) provider_hop_clusters: Vec, /// Sorted by site. pub(crate) peers: Vec, } @@ -161,7 +164,6 @@ where let sites: BTreeSet<&str> = candidates.iter().map(|candidate| candidate.site.as_str()).collect(); let mut addrs = remote_addrs(members, &sites, &overlay.local_site); if let Some(addr) = inputs.local_signals_addr - && sites.contains(overlay.local_site.as_str()) && certs::validate_site_name(&overlay.local_site).is_ok() { addrs.insert(overlay.local_site.clone(), addr.to_owned()); @@ -178,6 +180,7 @@ where window_secs: WINDOW_SECS, load_window_ms: load_window_ms(inputs.scrape_interval), candidates, + provider_hop_clusters: inputs.provider_hop_clusters.iter().cloned().collect(), peers, } } @@ -418,6 +421,15 @@ pub(crate) fn decide_write( } } +/// Withdrawals are safety-critical: do not let the ordinary write interval +/// leave an older serving route active after Grid has computed no candidates. +pub(crate) fn decide_empty_withdrawal(existing: Option<&str>, desired: &str) -> WriteDecision { + match existing { + Some(stored) if stored == desired => WriteDecision::Unchanged, + _ => WriteDecision::Write, + } +} + /// Last write time per `namespace/name`, for [`decide_write`]. #[derive(Debug, Default)] pub(crate) struct WriteGate { @@ -452,6 +464,8 @@ mod tests { /// No pins, as under SPIFFE trust. static NO_PINS: BTreeMap> = BTreeMap::new(); + /// No authenticated provider gateway in this fixture. + static NO_PROVIDER_HOPS: BTreeSet = BTreeSet::new(); const INPUTS: ServingInputs<'static> = ServingInputs { tls_mount: "/etc/praxis/tls", @@ -662,6 +676,10 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "table-driven test covers both irrelevant and unsafe peer addresses" + )] fn unsafe_or_irrelevant_peers_are_refused() { let cases = [ ("loopback", "127.0.0.1:9091"), @@ -720,6 +738,41 @@ mod tests { ); } + #[test] + fn empty_candidate_revision_keeps_peer_pollers_for_restoration() { + let inputs = ServingInputs { + local_signals_addr: Some("grid-operator-signals.grid.svc:9091"), + ..INPUTS + }; + let empty = render(&overlay(Vec::new()), [("site-b", "203.0.113.7:9091")], &inputs); + assert!(empty.candidates.is_empty()); + assert_eq!( + peer_sites(&empty), + [ + ("site-a", "grid-operator-signals.grid.svc:9091"), + ("site-b", "203.0.113.7:9091"), + ], + "no-route config retains metrics connections for route restoration" + ); + } + + #[test] + fn authenticated_gateway_candidates_carry_overlay_id_and_mtls_allowlist() { + let mut candidate = cand("llama", "site-b", "provider-b", None); + candidate.stable_id = Some("257a9450".to_owned()); + let hops = BTreeSet::from(["provider-b".to_owned()]); + let inputs = ServingInputs { + provider_hop_clusters: &hops, + ..INPUTS + }; + let config = render(&overlay(vec![candidate]), [], &inputs); + assert_eq!(config.provider_hop_clusters, ["provider-b"]); + assert_eq!(config.candidates[0].stable_id.as_deref(), Some("257a9450")); + let json = to_text(&config).expect("JSON"); + assert!(json.contains("\"provider_hop_clusters\": [\n \"provider-b\"")); + assert!(json.contains("\"stable_id\": \"257a9450\"")); + } + #[test] fn a_stale_duplicate_wins_in_either_order() { let mut stale = cand("llama", "site-b", "pool-b", None); @@ -898,6 +951,19 @@ mod tests { } } + #[test] + fn empty_withdrawal_bypasses_write_spacing_but_not_content_equality() { + assert_eq!( + decide_empty_withdrawal(Some("empty"), "empty"), + WriteDecision::Unchanged + ); + assert_eq!( + decide_empty_withdrawal(Some("old-route"), "empty"), + WriteDecision::Write + ); + assert_eq!(decide_empty_withdrawal(None, "empty"), WriteDecision::Write); + } + #[test] fn gate_remembers_writes_per_key() { let gate = WriteGate::default(); diff --git a/overlay-sync/src/watcher.rs b/overlay-sync/src/watcher.rs index fb9fe27c1..43c13c78b 100644 --- a/overlay-sync/src/watcher.rs +++ b/overlay-sync/src/watcher.rs @@ -561,6 +561,34 @@ mod tests { serde_json::to_string_pretty(&envelope).unwrap() } + fn valid_empty_envelope_json(scope: &ExpectedScope) -> String { + use crate::types::*; + let mut overlay = test_overlay(scope); + overlay.candidates.clear(); + let hex = overlay_digest(&overlay); + let envelope = OverlayEnvelope { + schema_version: "1.0.0".to_owned(), + revision: ContentRevision { + kind: "content_addressed".to_owned(), + algorithm: "sha256".to_owned(), + value: hex.clone(), + }, + content_digest: ContentDigest { + algorithm: "sha256".to_owned(), + value: hex, + }, + scope: OverlayScope { + network: scope.network.clone(), + gateway: scope.gateway.clone(), + namespace: scope.namespace.clone(), + local_site: scope.local_site.clone(), + }, + provenance: test_provenance(), + overlay, + }; + serde_json::to_string_pretty(&envelope).unwrap() + } + // -- restore_last_known_good ----------------------------------------------- #[test] @@ -631,6 +659,36 @@ mod tests { assert!(out.exists()); } + #[test] + fn process_configmap_writes_authoritative_empty_overlay_revision() { + let dir = tempfile::tempdir().unwrap(); + let out = dir.path().join("routing-overlay.json"); + let config = test_config(out.clone()); + let active = valid_envelope_json(&config.expected_scope); + let empty = valid_empty_envelope_json(&config.expected_scope); + let cm_for = |json| ConfigMap { + data: Some(BTreeMap::from([("routing-overlay.json".to_owned(), json)])), + ..Default::default() + }; + let status = SharedStatus::new("ns", "cm", "key"); + let metrics = Arc::new(Metrics::new()); + + assert_eq!( + process_configmap(&cm_for(active), &config, &status, &metrics), + ProcessOutcome::Written + ); + let prior_revision = status.written_revision().expect("active revision"); + assert_eq!( + process_configmap(&cm_for(empty), &config, &status, &metrics), + ProcessOutcome::Written + ); + let accepted_revision = status.written_revision().expect("empty revision"); + assert_ne!(accepted_revision, prior_revision); + let published: serde_json::Value = serde_json::from_slice(&std::fs::read(out).unwrap()).unwrap(); + assert!(published["overlay"]["candidates"].as_array().unwrap().is_empty()); + assert_eq!(accepted_revision, published["revision"]["value"].as_str().unwrap()); + } + #[test] fn process_configmap_rejects_missing_key() { let dir = tempfile::tempdir().unwrap(); diff --git a/tests/e2e/topologies/grid-provider-traffic/README.md b/tests/e2e/topologies/grid-provider-traffic/README.md index 9a1a89306..c1c35692f 100644 --- a/tests/e2e/topologies/grid-provider-traffic/README.md +++ b/tests/e2e/topologies/grid-provider-traffic/README.md @@ -128,13 +128,15 @@ The complete environment is in [`forge.yaml`](./forge.yaml). Supporting files ar Prerequisites include Docker, Kind, `kubectl`, Helm, OpenSSL, and `praxis-forge`. Build the `grid-operator` and Praxis AI gateway images before using the default `Never` pull policy. -Build `praxis-forge` and the two source images from clean Grid and Praxis AI checkouts: +Build `praxis-forge` and the source images from clean Grid and Praxis AI checkouts: ```console # From the `praxis-proxy/grid` repository. cargo build -p forge docker build -f deploy/operator/Containerfile \ -t grid-operator:provider-traffic-qualification . +docker build -f deploy/gateway/Containerfile \ + -t grid-gateway:provider-traffic-qualification . # From the Praxis AI repository. docker build -f Containerfile \ @@ -154,6 +156,7 @@ cargo test -p xtask provider_traffic --locked ```console export GRID_XTASK_OPERATOR_IMAGE=grid-operator:provider-traffic-qualification export GRID_XTASK_GATEWAY_IMAGE=praxis-ai:provider-traffic-qualification +export GRID_XTASK_GRID_GATEWAY_IMAGE=grid-gateway:provider-traffic-qualification export GRID_XTASK_SIM_IMAGE=ghcr.io/neuralmagic/vllm-vcr:vllm0.23 export GRID_XTASK_IMAGE_PULL_POLICY=Never @@ -163,9 +166,29 @@ cargo xtask env run-grid-provider-traffic-qualification \ --teardown ``` -The provider-traffic qualification has one complete six-scenario proof set. -Use `--full` for release qualification; `--quick` remains available for the -same focused proof when a bounded diagnostic run is preferred. +Both modes preserve the original six scenarios, including the 60-request +round-robin proof. `--full` additionally exercises the generated +`consumerConfig`, the embedded `grid-gateway` serving-config consumer, and the +provider-withdrawal lifecycle (fallback, a valid empty/no-route revision, +backend non-contact, and restoration). `--quick` runs only the original six +scenarios. + +The run-owned `GatewayRef`s explicitly set `supportsEmptyOverlay: true` because +the qualification builds both consumers from the compatible source under test. +Production deployments must first upgrade and roll out every consumer; the +field defaults to false so an older released image (including the chart's +default AI 0.4.0) retains its prior revision instead of receiving an invalid +empty snapshot. + +Full mode also exercises the generated consumer credential lifecycle in two +phases: it first renders `credential_inject` with an empty table and rolls out +the consumer with a read-only test-Secret mount at +`/run/secrets/grid-credentials/grid-system/vcr-inference-credential`; projected +Secret paths include namespace, Secret name, and key. Only after that rollout does it +set `supportsProjectedCredentials` and publish a credential-bearing Provider A +candidate. The in-cluster request omits the provider credential, and the probe +requires a successful provider-A-attributed response. The Secret value is not +included in test evidence. For registry-hosted images, use immutable tags or digests and set `GRID_XTASK_IMAGE_PULL_POLICY=IfNotPresent`. Do not reuse evidence from a run @@ -186,6 +209,14 @@ A qualifying run demonstrates: `provider-a`; - each provider serves exactly 20 requests; - the semantic overlay revision remains stable during traffic; +- in full mode, a partial withdrawal serves only the remaining providers; +- in full mode, last-route withdrawal publishes a new empty revision that the + overlay-backed and embedded Grid gateway consumers both accept and serve; +- new and previously bound overlay-backed requests receive an unattributed + no-route response, with provider workloads still Ready and backend counters + unchanged; +- restoration publishes and serves a new revision and attributed requests + resume; - teardown removes the clusters and shared network. Provider identity comes from request-scoped HTTP response attribution, not only diff --git a/tests/e2e/topologies/grid-provider-traffic/forge.yaml b/tests/e2e/topologies/grid-provider-traffic/forge.yaml index 9f7311736..3ce837b7b 100644 --- a/tests/e2e/topologies/grid-provider-traffic/forge.yaml +++ b/tests/e2e/topologies/grid-provider-traffic/forge.yaml @@ -230,6 +230,7 @@ spec: - name: consumer-gateway namespace: grid-system localSiteName: provider-a + supportsEmptyOverlay: true gridSite: name: provider-a region: provider-a @@ -293,6 +294,7 @@ spec: - name: consumer-gateway namespace: grid-system localSiteName: provider-b + supportsEmptyOverlay: true gridSite: name: provider-b region: provider-b @@ -356,6 +358,7 @@ spec: - name: consumer-gateway namespace: grid-system localSiteName: provider-c + supportsEmptyOverlay: true gridSite: name: provider-c region: provider-c @@ -492,6 +495,9 @@ spec: overlay: enabled: true existingConfigMap: "grid-overlay-grid-provider-traffic-consumer-gateway" + credentials: + - name: "vcr-inference-credential" + mountPath: "/run/secrets/grid-credentials/grid-system/vcr-inference-credential" tls: enabled: true existingSecret: "consumer-gateway-tls" diff --git a/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml b/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml index 6194e9bcc..3740cd6a8 100644 --- a/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml +++ b/tests/e2e/topologies/grid-provider-traffic/resources/common/vcr-provider-workload.yaml @@ -1,3 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vcr-inference-{{ cluster.name }}-config + namespace: grid-system +data: + config.yaml: | + port: 8000 + model: "Qwen/Qwen3-0.6B" + served-model-name: ["Qwen/Qwen3-0.6B"] + mode: "echo" + force-dummy-tokenizer: true + fake-metrics: + waiting-requests: 0 + kv-cache-usage: 0.0 +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -18,6 +34,7 @@ spec: app.kubernetes.io/name: vllm-vcr app.kubernetes.io/instance: "{{ cluster.name }}" spec: + terminationGracePeriodSeconds: 15 automountServiceAccountToken: false securityContext: runAsNonRoot: true @@ -29,6 +46,13 @@ spec: - name: vcr image: "{{ cluster.properties.vcrImage }}" imagePullPolicy: "{{ cluster.properties.imagePullPolicy }}" + args: + - --config + - /config/config.yaml + - --logtostderr=true + volumeMounts: + - name: config + mountPath: /config securityContext: allowPrivilegeEscalation: false capabilities: @@ -97,6 +121,10 @@ spec: memory: 512Mi limits: memory: 2Gi + volumes: + - name: config + configMap: + name: vcr-inference-{{ cluster.name }}-config --- apiVersion: v1 kind: Service diff --git a/xtask/src/env/certs.rs b/xtask/src/env/certs.rs index 587b4d301..153b04749 100644 --- a/xtask/src/env/certs.rs +++ b/xtask/src/env/certs.rs @@ -50,11 +50,22 @@ const WRONG_ORG_CERT_NAME: &str = "wrong-org-client"; /// /// Returns an error if certificate generation or file writes fail. pub(crate) fn generate_all(cluster_names: &[String]) -> Result> { - let dir = PathBuf::from(CERTS_DIR); - std::fs::create_dir_all(&dir)?; + generate_all_in_dir(cluster_names, Path::new(CERTS_DIR)) +} + +/// Generate a CA and per-cluster certificates in a caller-owned directory. +/// +/// This is used by qualifications that must keep certificates isolated from +/// concurrent runs. Existing files in `dir` are reused only within that run. +/// +/// # Errors +/// +/// Returns an error if certificate generation or file writes fail. +pub(crate) fn generate_all_in_dir(cluster_names: &[String], dir: &Path) -> Result> { + std::fs::create_dir_all(dir)?; let ca_was_complete = dir.join("ca.pem").exists() && dir.join("ca-key.pem").exists(); - let ca = load_or_generate_ca(&dir)?; + let ca = load_or_generate_ca(dir)?; for name in cluster_names { if ca_was_complete && identity_exists(&dir, name) && names_its_site(&dir, name)? { @@ -68,9 +79,9 @@ pub(crate) fn generate_all(cluster_names: &[String]) -> Result Result, Box Result> { - certificate_sha256(&Path::new(CERTS_DIR).join(format!("{site}-cert.pem"))) + site_certificate_fingerprint_in_dir(site, Path::new(CERTS_DIR)) +} + +/// Compute the canonical fingerprint for a site certificate in a caller-owned directory. +pub(crate) fn site_certificate_fingerprint_in_dir( + site: &str, + certs_dir: &Path, +) -> Result> { + certificate_sha256(&certs_dir.join(format!("{site}-cert.pem"))) } /// Compute the canonical fingerprint from a PEM certificate string. @@ -421,6 +440,17 @@ mod tests { } } + #[test] + fn site_fingerprint_uses_the_caller_owned_certificate_directory() { + let test_dir = tempfile::tempdir().unwrap_or_else(|_| std::process::abort()); + let sites = vec!["provider-b".to_owned()]; + generate_all_in_dir(&sites, test_dir.path()).unwrap_or_else(|_| std::process::abort()); + + let fingerprint = site_certificate_fingerprint_in_dir("provider-b", test_dir.path()) + .unwrap_or_else(|_| std::process::abort()); + assert_eq!(fingerprint.len(), 64); + } + #[test] fn write_pem_creates_file() { let test_dir = tempfile::tempdir().unwrap_or_else(|_| std::process::abort()); diff --git a/xtask/src/env/forge_config.rs b/xtask/src/env/forge_config.rs index ebc205a47..2c4cf8774 100644 --- a/xtask/src/env/forge_config.rs +++ b/xtask/src/env/forge_config.rs @@ -68,7 +68,7 @@ pub(crate) fn materialize_with_images( clippy::too_many_lines, reason = "The bounded image-property rewrite is easiest to audit as one operation." )] -fn apply_image_values( +pub(crate) fn apply_image_values( config: &mut serde_yaml::Value, images: &ImageOverrides, ) -> Result<(), Box> { @@ -152,6 +152,39 @@ mod tests { ); } + #[test] + #[expect( + clippy::expect_used, + clippy::indexing_slicing, + reason = "the rendered Forge contract is a fixed qualification fixture" + )] + fn image_overrides_include_overlay_sync_for_every_cluster() { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../tests/e2e/topologies"); + let images = super::ImageOverrides { + gateway: "praxis-ai:run".to_owned(), + operator: "grid-operator:run".to_owned(), + overlay_sync: "grid-overlay-sync:run".to_owned(), + vcr: "llm-d-inference-sim:run".to_owned(), + pull_policy: "Never".to_owned(), + }; + + for topology in ["grid-provider-traffic", "grid-static-weighted"] { + let source = root.join(topology).join("forge.yaml"); + let content = fs::read_to_string(source).expect("read qualification Forge config"); + let mut config: serde_yaml::Value = serde_yaml::from_str(&content).expect("parse Forge config"); + super::apply_image_values(&mut config, &images).expect("apply image overrides"); + let clusters = config["spec"]["clusters"].as_sequence().expect("clusters"); + assert_eq!(clusters.len(), 3); + for cluster in clusters { + let properties = &cluster["properties"]; + assert_eq!(properties["overlaySyncImage"].as_str(), Some("grid-overlay-sync:run")); + assert_eq!(properties["overlaySyncImageRepo"].as_str(), Some("grid-overlay-sync")); + assert_eq!(properties["overlaySyncImageTag"].as_str(), Some("run")); + assert_eq!(properties["imagePullPolicy"].as_str(), Some("Never")); + } + } + } + #[test] #[expect( clippy::expect_used, diff --git a/xtask/src/env/image_overrides.rs b/xtask/src/env/image_overrides.rs index 993e71bfe..d9354addf 100644 --- a/xtask/src/env/image_overrides.rs +++ b/xtask/src/env/image_overrides.rs @@ -5,7 +5,10 @@ //! release images; local development remains available through explicit //! overrides. -use std::env; +use std::{ + env, + process::{Command, Stdio}, +}; use super::IngressMode; @@ -158,6 +161,42 @@ pub(crate) fn should_skip_kind_image_loading() -> bool { image_pull_policy() != "Never" } +/// Import a host image into a run-owned Kind node using only its linux/amd64 +/// image content. This avoids OCI-index imports failing when the local Docker +/// store has only the linux/amd64 child content. +pub(crate) fn load_docker_image_into_kind(image: &str, kind_name: &str) -> Result<(), Box> { + let control_plane = format!("{kind_name}-control-plane"); + let mut save = Command::new("docker") + .args(["save", "--platform", "linux/amd64", image]) + .stdout(Stdio::piped()) + .spawn()?; + let save_stdout = save.stdout.take().ok_or("docker save did not provide stdout")?; + let import_status = Command::new("docker") + .args([ + "exec", + "--privileged", + "-i", + &control_plane, + "ctr", + "--namespace=k8s.io", + "images", + "import", + "--digests", + "--snapshotter=overlayfs", + "-", + ]) + .stdin(save_stdout) + .status()?; + let save_status = save.wait()?; + if !save_status.success() { + return Err(format!("docker save failed for {image}").into()); + } + if !import_status.success() { + return Err(format!("failed to import {image} into {control_plane}").into()); + } + Ok(()) +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- diff --git a/xtask/src/env/provider_traffic_qualification.rs b/xtask/src/env/provider_traffic_qualification.rs index 4e046be04..eb3f68658 100644 --- a/xtask/src/env/provider_traffic_qualification.rs +++ b/xtask/src/env/provider_traffic_qualification.rs @@ -2,13 +2,19 @@ use std::{ collections::{BTreeMap, BTreeSet}, fs, + io::Write as _, + net::{TcpListener, TcpStream}, path::{Path, PathBuf}, - process::Command, - sync::atomic::{AtomicU64, Ordering}, + process::{Child, Command, Stdio}, + sync::{ + OnceLock, + atomic::{AtomicU64, Ordering}, + }, time::{Duration, Instant}, }; use serde::Serialize; +use sha2::{Digest as _, Sha256}; use super::{DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, safe_truncate_str}; @@ -16,9 +22,6 @@ use super::{DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, safe_trunca // Constants // ----------------------------------------------------------------------------- -/// Directory where generated TLS certificates are stored. -const CERTS_DIR: &str = "tests/env/certs"; - /// Ordered provider-site cluster names in the provider-traffic scenario. /// /// The consumer entrypoint is deployed only in `CONSUMER_SITE`; the other @@ -37,21 +40,49 @@ const EVIDENCE_SCHEMA_VERSION: &str = "1"; /// Kubernetes namespace for all Grid components. const GRID_SYSTEM_NS: &str = "grid-system"; +/// `GridNetwork` name in the focused provider-traffic topology. +const GRID_NETWORK_NAME: &str = "grid-provider-traffic"; + +/// Additional startup-configured grid-gateway consumer used to qualify serving-config reload. +const GRID_SERVING_GATEWAY: &str = "grid-serving-gateway"; + +/// Operator `ConfigMap` consumed by the embedded `grid-gateway` filter. +const GRID_SERVING_CONFIGMAP: &str = "grid-serving-grid-provider-traffic-grid-serving-gateway"; + /// Overlay `ConfigMap` name created by the Grid operator for consumer gateways. const OVERLAY_CONFIGMAP: &str = "grid-overlay-grid-provider-traffic-consumer-gateway"; +/// Physical Kind cluster prefix assigned to this qualification run. +static RUN_CLUSTER_PREFIX: OnceLock = OnceLock::new(); + /// Provider credential secret name (matches Helm `credentials[0].name`). const VCR_INFERENCE_CREDENTIAL: &str = "vcr-inference-credential"; /// Stable terminal separator that also remains readable in captured logs. const OUTPUT_RULE: &str = "==============================================================================="; +/// Marker appended by curl after the response body so probes do not parse HTTP header formatting. +const CURL_HTTP_STATUS_MARKER: &str = "GRID258_HTTP_STATUS:"; + +/// Maximum wait for Grid to publish a credential-bearing test overlay. +const PROJECTED_CREDENTIAL_OVERLAY_TIMEOUT: Duration = Duration::from_secs(180); + /// Provider gateway service name advertised via SWIM for cross-site discovery. const PROVIDER_GATEWAY_SERVICE: &str = "provider-gateway"; /// Provider gateway port advertised via SWIM for cross-site discovery. const PROVIDER_GATEWAY_PORT: &str = "8443"; +/// Label used by the run-owned `InferenceProvider` selectors. +const PROVIDER_SITE_LABEL: &str = "grid.praxis-proxy.io/provider-site"; + +/// Provider `InferenceProvider` resources created by this topology, in origin-cluster order. +const PROVIDER_RESOURCES: &[(&str, &str)] = &[ + ("provider-a", "vcr-provider-a-provider"), + ("provider-b", "vcr-provider-b-provider"), + ("provider-c", "vcr-provider-c-provider"), +]; + /// Provider gateway TLS secret name (matches Helm `existingSecret` reference). const PROVIDER_TLS_SECRET: &str = "provider-gateway-tls"; @@ -75,10 +106,46 @@ struct ProviderTrafficContext { demo_root: PathBuf, /// Path to the resolved Forge config. resolved_config: PathBuf, + /// Run-owned Forge state directory. + forge_state_dir: PathBuf, + /// Run-owned TLS material directory. + certs_dir: PathBuf, + /// Run-owned evidence directory. + evidence_dir: PathBuf, /// Path to the forge binary. forge_bin: PathBuf, } +/// Own a diagnostic-only backend metrics port-forward and stop it on every exit path. +struct BackendMetricsPortForward { + /// Child process serving the local loopback tunnel. + child: Option, +} + +impl Drop for BackendMetricsPortForward { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _kill = child.kill(); + let _wait = child.wait(); + } + } +} + +/// Build the context name for one cluster in this qualification run. +fn cluster_context(cluster: &str) -> String { + let prefix = RUN_CLUSTER_PREFIX.get().map_or("grid-provider-traffic", String::as_str); + format!("kind-{prefix}-{cluster}") +} + +/// Resolve Grid chart paths from the xtask crate's workspace root, not from a +/// topology directory or the caller's current working directory. +fn grid_repository_root() -> Result> { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(Path::to_path_buf) + .ok_or_else(|| "xtask manifest directory has no Grid repository parent".into()) +} + // ----------------------------------------------------------------------------- // Overlay State // ----------------------------------------------------------------------------- @@ -118,6 +185,18 @@ struct OverlayCandidate { admission_state: Option, /// Explicit priority group emitted by the operator, when present. selection_group: Option, + /// Whether this overlay candidate carries a provider Secret reference. + #[serde(default)] + has_credential_reference: bool, +} + +/// Sanitized result of one request used by the no-route lifecycle proof. +#[derive(Clone, Debug, Serialize)] +struct LifecycleRequest { + /// Observed HTTP status. + status: u16, + /// Provider gateway attribution, absent when no provider served the request. + provider: Option, } /// Pre- and post-SWIM overlay snapshots for evidence. @@ -436,11 +515,69 @@ fn response_header(output: &[u8], name: &str) -> Option { }) } +/// Parse curl's explicit HTTP status marker, rejecting transport failures (`000`). +fn curl_http_status(output: &[u8]) -> Result { + let response = String::from_utf8_lossy(output); + let status = response + .lines() + .find_map(|line| line.strip_prefix(CURL_HTTP_STATUS_MARKER)) + .ok_or_else(|| "curl output did not contain its HTTP status marker".to_owned())? + .parse::() + .map_err(|error| format!("curl emitted an invalid HTTP status: {error}"))?; + if status == 0 { + return Err("curl received no HTTP response (status 000)".to_owned()); + } + Ok(status) +} + +/// Remove ANSI CSI styling so tracing fields remain parseable in terminal output. +fn strip_ansi_csi(input: &str) -> String { + let mut output = Vec::with_capacity(input.len()); + let mut bytes = input.bytes().peekable(); + while let Some(byte) = bytes.next() { + if byte == 0x1B && bytes.peek() == Some(&b'[') { + bytes.next(); + for control in bytes.by_ref() { + if (0x40..=0x7E).contains(&control) { + break; + } + } + } else { + output.push(byte); + } + } + String::from_utf8_lossy(&output).into_owned() +} + +/// Return every exact serving-revision log observation after normalizing ANSI styling. +fn embedded_serving_revision_lines(logs: &str, digest: &str, count: u64) -> Vec { + let clean = strip_ansi_csi(logs); + clean + .lines() + .filter(|line| { + line.contains("grid_serving_revision_serving") + && line.contains(digest) + && line.contains(&format!("candidate_count={count}")) + }) + .map(str::to_owned) + .collect() +} + +/// Require an exact serving event beyond those already observed for this digest. +fn has_new_embedded_serving_revision_observation( + logs: &str, + digest: &str, + count: u64, + prior_matching_observations: usize, +) -> bool { + embedded_serving_revision_lines(logs, digest, count).len() > prior_matching_observations +} + /// Wait for the demo environment to be ready. fn wait_for_environment_ready() -> Result> { // Wait for Grid operators to converge for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); eprintln!(" [WAIT] {cluster}: Grid operator convergence"); // Wait for deployment to be ready @@ -474,7 +611,7 @@ fn assert_cluster_health() -> AssertionResult { let mut cluster_health = Vec::new(); for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); // Check API server responsiveness let api_start = Instant::now(); @@ -559,7 +696,7 @@ fn assert_component_deployment() -> AssertionResult { let static_components = ["grid-operator", "provider-gateway"]; for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let mock_name = format!("vcr-inference-{cluster}"); let cluster_components: Vec<&str> = static_components .iter() @@ -663,7 +800,7 @@ fn assert_swim_convergence() -> AssertionResult { let mut all_converged = true; for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let status = poll_until( || { let output = Command::new("kubectl") @@ -741,7 +878,7 @@ fn assert_site_auto_discovery() -> AssertionResult { let mut all_ok = true; for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let result = poll_until( || { @@ -837,7 +974,7 @@ fn assert_overlay_acceptance() -> AssertionResult { { let cluster = CONSUMER_SITE; - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); // Step 1: Overlay ConfigMap exists let overlay_output = Command::new("kubectl") @@ -967,8 +1104,8 @@ fn assert_overlay_acceptance() -> AssertionResult { fn require_local_image(image: &str) -> Result<(), Box> { let status = Command::new("docker") .args(["image", "inspect", image]) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) .status()?; if status.success() { return Ok(()); @@ -985,11 +1122,7 @@ fn require_local_image(image: &str) -> Result<(), Box> { /// Reads image references from the `GRID_XTASK_*_IMAGE` environment variables /// (same source as `apply_image_overrides`). When `imagePullPolicy` is not /// `Never`, this is a no-op. -#[expect( - clippy::too_many_lines, - reason = "Image loading is one bounded setup operation across the three clusters." -)] -fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result<(), Box> { +fn load_images_into_clusters() -> Result<(), Box> { let pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); if pull_policy != "Never" { eprintln!(" skipping Kind image loading (pull policy is {pull_policy})"); @@ -1001,27 +1134,23 @@ fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result let operator = std::env::var("GRID_XTASK_OPERATOR_IMAGE") .unwrap_or_else(|_| "grid-operator:provider-traffic-qualification".to_owned()); let vcr = crate::env::image_overrides::sim_image(); + let overlay_sync = crate::env::image_overrides::overlay_sync_image(); - for image in [&gateway, &operator, &vcr] { + for image in [&gateway, &operator, &vcr, &overlay_sync] { require_local_image(image)?; eprintln!(" verified local image: {image}"); } for cluster in CLUSTERS { - for image in [&gateway, &operator, &vcr] { + for image in [&gateway, &operator, &vcr, &overlay_sync] { eprintln!(" loading {image} into {cluster}..."); - let output = Command::new(forge_bin.as_os_str()) - .arg("--config") - .arg(resolved_config) - .args(["--non-interactive", "cluster", "load-image", cluster, image]) - .output()?; - if !output.status.success() { - return Err(format!( - "failed to load {image} into {cluster}: {}", - String::from_utf8_lossy(&output.stderr).trim() - ) - .into()); - } + let kind_name = format!( + "{}-{cluster}", + RUN_CLUSTER_PREFIX + .get() + .ok_or("run-owned cluster prefix was not initialized")? + ); + crate::env::image_overrides::load_docker_image_into_kind(image, &kind_name)?; } eprintln!(" [OK] {cluster}: all images loaded"); } @@ -1032,12 +1161,12 @@ fn load_images_into_clusters(forge_bin: &Path, resolved_config: &Path) -> Result /// /// Must be called BEFORE `forge up` so the certificates exist on the host /// when `install_provider_boundary` creates the Kubernetes Secrets. -fn stage_provider_boundary() -> Result<(), Box> { +fn stage_provider_boundary(certs_dir: &Path) -> Result<(), Box> { let identities: Vec = CLUSTERS.iter().map(|c| (*c).to_owned()).collect(); - certs::generate_all(&identities)?; + certs::generate_all_in_dir(&identities, certs_dir)?; let wrong_ca = ::certs::generate_ca("Combined Site untrusted test CA")?; - fs::write(Path::new(CERTS_DIR).join("untrusted-ca.pem"), wrong_ca.cert_pem)?; + fs::write(certs_dir.join("untrusted-ca.pem"), wrong_ca.cert_pem)?; eprintln!(" [OK] TLS certificates generated for provider-a, provider-b, provider-c"); Ok(()) @@ -1047,11 +1176,9 @@ fn stage_provider_boundary() -> Result<(), Box> { /// /// Must be called AFTER `forge up` since the clusters must exist. Gateway /// deployments are restarted so pods pick up the new volume mounts. -fn install_provider_boundary() -> Result<(), Box> { - let certs_dir = Path::new(CERTS_DIR); - +fn install_provider_boundary(certs_dir: &Path) -> Result<(), Box> { for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); apply_tls_secret(&context, cluster, CONSUMER_TLS_SECRET, certs_dir)?; apply_tls_secret(&context, cluster, PROVIDER_TLS_SECRET, certs_dir)?; @@ -1140,7 +1267,7 @@ fn apply_credential_secret(context: &str, secret_name: &str, token: &str) -> Res reason = "The reader validates and parses one Kubernetes ConfigMap response." )] fn read_cluster_overlay(cluster: &str) -> Result> { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let output = Command::new("kubectl") .args([ @@ -1206,6 +1333,7 @@ fn read_cluster_overlay(cluster: &str) -> Result Result Result Result, Box> { - let context = "kind-grid-provider-traffic-provider-a"; + let context = cluster_context("provider-a"); let replicas = Command::new("kubectl") .args([ "--context", - context, + &context, "-n", GRID_SYSTEM_NS, "get", @@ -1267,7 +1396,7 @@ fn wait_for_round_robin_readiness() -> Result Result Result> { + let context = cluster_context("provider-a"); let output = Command::new("kubectl") .args([ "--context", - "kind-grid-provider-traffic-provider-a", + &context, "-n", GRID_SYSTEM_NS, "logs", @@ -1628,7 +1758,7 @@ fn collect_overlay_diagnostics() { eprintln!(" [DIAG] Collecting overlay failure diagnostics...\n"); for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); eprintln!(" ======== {cluster} ========"); eprintln!(" [DIAG] {cluster}: 1. Operator deployment SWIM env vars"); @@ -1791,12 +1921,12 @@ fn collect_overlay_diagnostics() { eprintln!(" [DIAG] 10. Cross-cluster SWIM connectivity check"); for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); for target in CLUSTERS { if *target == *cluster { continue; } - let target_context = format!("kind-grid-provider-traffic-{target}"); + let target_context = cluster_context(target); if let Ok(output) = Command::new("kubectl") .args([ "--context", @@ -1875,7 +2005,7 @@ fn materialize_provider_config( .replace("SITE_PLACEHOLDER", cluster) .replace("CANDIDATE_ID_PLACEHOLDER", stable_id); - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let create = Command::new("kubectl") .args([ @@ -1908,65 +2038,155 @@ fn materialize_provider_config( Ok(()) } -/// Materialize the Forge configuration with image overrides. -fn materialize_config(source: &Path) -> Result> { +/// Materialize a run-owned Forge config beside its source so topology-relative +/// resource paths continue to resolve from the documented directory. +fn materialize_config( + source: &Path, + run_id: &str, + cluster_prefix: &str, + state_dir: &Path, +) -> Result> { let content = fs::read_to_string(source)?; let mut config: serde_yaml::Value = serde_yaml::from_str(&content)?; - apply_image_overrides(&mut config); + apply_image_overrides(&mut config)?; + scope_environment_name(&mut config, run_id)?; + let runtime = config + .get_mut("spec") + .and_then(|spec| spec.get_mut("runtime")) + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or("Forge config is missing spec.runtime")?; + runtime.insert( + serde_yaml::Value::String("clusterPrefix".to_owned()), + serde_yaml::Value::String(cluster_prefix.to_owned()), + ); + rewrite_run_scoped_paths(&mut config, cluster_prefix, state_dir); let rendered = serde_yaml::to_string(&config)?; let parent = source.parent().ok_or("source config must have parent directory")?; - let output = parent.join(".forge.resolved.yaml"); + let output = parent.join(format!(".forge.resolved-{run_id}.yaml")); fs::write(&output, rendered)?; Ok(output) } -/// Apply image overrides from environment variables to the Forge configuration. -#[expect( - clippy::too_many_lines, - clippy::collapsible_if, - reason = "Image override application with structured YAML manipulation; nested ifs follow YAML structure hierarchy" -)] -fn apply_image_overrides(config: &mut serde_yaml::Value) { - let gateway_image = std::env::var("GRID_XTASK_GATEWAY_IMAGE") - .unwrap_or_else(|_| "praxis-ai:provider-traffic-qualification".to_owned()); - let operator_image = std::env::var("GRID_XTASK_OPERATOR_IMAGE") - .unwrap_or_else(|_| "grid-operator:provider-traffic-qualification".to_owned()); - let vcr_image = crate::env::image_overrides::sim_image(); - let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); +/// Give Forge's environment and derived Docker network a unique identity. +fn scope_environment_name(config: &mut serde_yaml::Value, run_id: &str) -> Result<(), Box> { + let metadata = config + .get_mut("metadata") + .and_then(serde_yaml::Value::as_mapping_mut) + .ok_or("Forge config is missing metadata")?; + let name_key = serde_yaml::Value::String("name".to_owned()); + let name = metadata + .get(&name_key) + .and_then(serde_yaml::Value::as_str) + .ok_or("Forge config metadata is missing name")?; + let scoped_name = format!("{name}-{run_id}"); + if scoped_name.len() > 63 { + return Err(format!("run-scoped Forge environment name is too long: {scoped_name}").into()); + } + metadata.insert(name_key, serde_yaml::Value::String(scoped_name)); + Ok(()) +} - let (gateway_repo, gateway_tag) = parse_image_ref(&gateway_image); - let (operator_repo, operator_tag) = parse_image_ref(&operator_image); +/// Rewrite fixed Kind contexts and `exec` paths to use this run's Forge state. +/// `template-file` targets intentionally keep their `.forge/runtime/` prefix: +/// Forge resolves those relative to `--state-dir` itself. +fn rewrite_run_scoped_paths(value: &mut serde_yaml::Value, cluster_prefix: &str, state_dir: &Path) { + rewrite_cluster_contexts(value, cluster_prefix); + rewrite_exec_state_references(value, state_dir); +} - if let Some(spec) = config.get_mut("spec") { - if let Some(clusters) = spec.get_mut("clusters") { - if let Some(clusters_array) = clusters.as_sequence_mut() { - for cluster in clusters_array { - if let Some(properties) = cluster.get_mut("properties") { - if let Some(props_map) = properties.as_mapping_mut() { - let pairs = [ - ("gatewayImage", &gateway_image), - ("operatorImage", &operator_image), - ("vcrImage", &vcr_image), - ("imagePullPolicy", &image_pull_policy), - ("gatewayImageRepo", &gateway_repo), - ("gatewayImageTag", &gateway_tag), - ("operatorImageRepo", &operator_repo), - ("operatorImageTag", &operator_tag), - ]; - for (key, val) in pairs { - props_map.insert( - serde_yaml::Value::String(key.to_owned()), - serde_yaml::Value::String(val.clone()), - ); - } - } - } +/// Scope hard-coded Kind context names without changing logical cluster names. +fn rewrite_cluster_contexts(value: &mut serde_yaml::Value, cluster_prefix: &str) { + match value { + serde_yaml::Value::String(string) => { + *string = string.replace("kind-grid-provider-traffic-", &format!("kind-{cluster_prefix}-")); + }, + serde_yaml::Value::Sequence(sequence) => { + for child in sequence { + rewrite_cluster_contexts(child, cluster_prefix); + } + }, + serde_yaml::Value::Mapping(mapping) => { + for child in mapping.values_mut() { + rewrite_cluster_contexts(child, cluster_prefix); + } + }, + serde_yaml::Value::Null + | serde_yaml::Value::Bool(_) + | serde_yaml::Value::Number(_) + | serde_yaml::Value::Tagged(_) => {}, + } +} + +/// Rewrite `.forge/runtime/` only inside shell exec commands. Forge's own +/// `template-file` step maps that prefix into `/runtime`; shell +/// commands run from the repository root and must use the corresponding path. +fn rewrite_exec_state_references(value: &mut serde_yaml::Value, state_dir: &Path) { + match value { + serde_yaml::Value::Mapping(mapping) => { + let is_exec = mapping + .get(serde_yaml::Value::String("type".to_owned())) + .and_then(serde_yaml::Value::as_str) + == Some("exec"); + if is_exec { + if let Some(command) = mapping.get_mut(serde_yaml::Value::String("command".to_owned())) { + rewrite_state_path_strings(command, state_dir); + } + } else { + for child in mapping.values_mut() { + rewrite_exec_state_references(child, state_dir); } } - } + }, + serde_yaml::Value::Sequence(sequence) => { + for child in sequence { + rewrite_exec_state_references(child, state_dir); + } + }, + serde_yaml::Value::Null + | serde_yaml::Value::Bool(_) + | serde_yaml::Value::Number(_) + | serde_yaml::Value::String(_) + | serde_yaml::Value::Tagged(_) => {}, + } +} + +/// Rewrite generated Forge runtime paths to the current run's isolated state directory. +fn rewrite_state_path_strings(value: &mut serde_yaml::Value, state_dir: &Path) { + match value { + serde_yaml::Value::String(string) => { + *string = string.replace(".forge/runtime/", &format!("{}/runtime/", state_dir.display())); + }, + serde_yaml::Value::Sequence(sequence) => { + for child in sequence { + rewrite_state_path_strings(child, state_dir); + } + }, + serde_yaml::Value::Mapping(mapping) => { + for child in mapping.values_mut() { + rewrite_state_path_strings(child, state_dir); + } + }, + serde_yaml::Value::Null + | serde_yaml::Value::Bool(_) + | serde_yaml::Value::Number(_) + | serde_yaml::Value::Tagged(_) => {}, } } +/// Apply the shared Forge image overrides, including the overlay-sync image. +fn apply_image_overrides(config: &mut serde_yaml::Value) -> Result<(), Box> { + let images = crate::env::forge_config::ImageOverrides { + gateway: std::env::var("GRID_XTASK_GATEWAY_IMAGE") + .unwrap_or_else(|_| "praxis-ai:provider-traffic-qualification".to_owned()), + operator: std::env::var("GRID_XTASK_OPERATOR_IMAGE") + .unwrap_or_else(|_| "grid-operator:provider-traffic-qualification".to_owned()), + overlay_sync: crate::env::image_overrides::overlay_sync_image(), + vcr: crate::env::image_overrides::sim_image(), + pull_policy: std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()), + }; + crate::env::forge_config::apply_image_values(config, &images) +} + /// Parse image reference into (repo, tag) components. fn parse_image_ref(image: &str) -> (String, String) { if let Some(colon_pos) = image.rfind(':') { @@ -1980,11 +2200,22 @@ fn parse_image_ref(image: &str) -> (String, String) { } /// Prepare setup context from configuration. -fn prepare_setup(forge_config: &Path) -> Result> { +fn prepare_setup( + forge_config: &Path, + evidence_dir: &Path, + run_id: &str, + cluster_prefix: &str, +) -> Result> { let root = super::demo_root(forge_config); eprintln!("Forge config: {}", forge_config.display()); eprintln!("Demo root: {}", root.display()); - let resolved_config = materialize_config(forge_config)?; + let evidence_dir = fs::canonicalize(evidence_dir)?; + let forge_state_dir = evidence_dir.join("forge-state"); + let certs_dir = evidence_dir.join("certs"); + fs::create_dir_all(&forge_state_dir)?; + fs::create_dir_all(&certs_dir)?; + let resolved_config = materialize_config(forge_config, run_id, cluster_prefix, &forge_state_dir)?; + fs::copy(&resolved_config, evidence_dir.join("resolved-forge.yaml"))?; let forge_bin = glb::resolve_forge_binary() .ok_or("praxis-forge binary not found")? .into(); @@ -1992,22 +2223,36 @@ fn prepare_setup(forge_config: &Path) -> Result Command { + let config = context.resolved_config.display().to_string(); + let state_dir = context.forge_state_dir.display().to_string(); + let mut command = Command::new(&context.forge_bin); + command + .args(["--config", &config, "--non-interactive", "--state-dir", &state_dir]) + .env("FORGE_STATE_DIR", &context.forge_state_dir); + command +} + /// Authorize auto-discovered remote `GridSites` with identity trust material. /// /// For each local cluster, waits for the two remote auto-discovered `GridSites`, /// verifies the SWIM-advertised certificate matches the staged identity, then /// patches `spec.egress.tls.serverName` and `spec.trust.canonicalFingerprints`. /// The controller transitions the site to Active naturally after the patch. -fn authorize_discovered_sites() -> Result<(), Box> { +fn authorize_discovered_sites(certs_dir: &Path) -> Result<(), Box> { const TRUST_TIMEOUT: Duration = Duration::from_secs(120); const GRID_NETWORK: &str = "grid-provider-traffic"; for local in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{local}"); + let context = cluster_context(local); eprintln!(); eprintln!(" {local}: authorizing remote provider sites"); for remote in CLUSTERS { @@ -2016,7 +2261,7 @@ fn authorize_discovered_sites() -> Result<(), Box> { } let site_name = format!("{GRID_NETWORK}-{remote}"); operator::wait_for_auto_gridsite(&context, &site_name, GRID_NETWORK, TRUST_TIMEOUT)?; - let canonical_fp = certs::site_certificate_fingerprint(remote)?; + let canonical_fp = certs::certificate_sha256(&certs_dir.join(format!("{remote}-cert.pem")))?; operator::wait_for_expected_site_certificate(&context, &site_name, &canonical_fp, TRUST_TIMEOUT)?; let server_name = format!("{remote}.grid.internal"); operator::patch_gridsite_identity_trust(&context, &site_name, &canonical_fp, &server_name)?; @@ -2048,10 +2293,7 @@ fn deploy_setup(context: &ProviderTrafficContext) -> Result Result Result Result Result<(), Box> { + let apply_stack = |cluster: &str, stack: &str| -> Result<(), Box> { eprintln!(" applying {stack} to {cluster}..."); - let stack_status = Command::new(forge_bin) - .arg("--config") - .arg(resolved_config) - .args(["--non-interactive", "stack", "apply", cluster, stack]) + let stack_status = forge_command(context) + .args(["stack", "apply", cluster, stack]) .status()?; if !stack_status.success() { capture_stack_failure(cluster, stack); @@ -2120,11 +2353,11 @@ fn deploy_setup(context: &ProviderTrafficContext) -> Result Result Result Result Result Result Result Result Result BTreeMap { - let mut results = BTreeMap::new(); - let mut scenario_num: usize = 0; - let mut scenario = || { - scenario_num += 1; - scenario_num - }; - - eprintln!(); - eprintln!("=== PROVIDER TRAFFIC SCENARIOS ==="); - eprintln!(); - - eprintln!("[SCENARIO {}] Verify three provider clusters are healthy", scenario()); - run_and_insert(&mut results, "cluster_health", assert_cluster_health); - - eprintln!(); - eprintln!("[SCENARIO {}] Verify component deployment", scenario()); - run_and_insert(&mut results, "component_deployment", assert_component_deployment); - - eprintln!(); - eprintln!("[SCENARIO {}] Verify SWIM convergence", scenario()); - run_and_insert(&mut results, "swim_convergence", assert_swim_convergence); +/// Return the scenario contract for one qualification mode. +fn provider_traffic_scenario_names(mode: DemoMode) -> Vec<&'static str> { + let mut names = vec![ + "cluster_health", + "component_deployment", + "swim_convergence", + "site_auto_discovery", + "overlay_acceptance", + "provider_gateway_round_robin", + ]; + if mode == DemoMode::Full { + names.push("generated_consumer_config_convergence"); + names.push("embedded_serving_gateway_convergence"); + names.push("provider_withdrawal_lifecycle"); + } + names +} - eprintln!(); - eprintln!("[SCENARIO {}] Verify site auto-discovery", scenario()); - run_and_insert(&mut results, "site_auto_discovery", assert_site_auto_discovery); +/// Read the provider gateway's run-owned `LoadBalancer` IP for the generated +/// consumer endpoint inventory. +fn read_provider_gateway_address(site: &str) -> Result> { + let context = cluster_context(site); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "service", + "provider-gateway", + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err(format!("{site}: could not read provider-gateway Service address").into()); + } + let service: serde_json::Value = serde_json::from_slice(&output.stdout)?; + let address = service + .pointer("/status/loadBalancer/ingress/0/ip") + .and_then(serde_json::Value::as_str) + .or_else(|| { + service + .pointer("/status/loadBalancer/ingress/0/hostname") + .and_then(serde_json::Value::as_str) + }) + .filter(|address| !address.trim().is_empty()) + .ok_or_else(|| format!("{site}: provider-gateway Service has no LoadBalancer address"))?; + Ok(format!("{address}:{PROVIDER_GATEWAY_PORT}")) +} - eprintln!(); - eprintln!("[SCENARIO {}] Verify overlay acceptance", scenario()); - run_and_insert(&mut results, "overlay_acceptance", assert_overlay_acceptance); +/// Build the endpoint list for all providers while retaining the topology's +/// verified TLS identities. +fn generated_consumer_endpoints() -> Result, Box> { + PROVIDER_RESOURCES + .iter() + .map(|(site, resource)| { + Ok(serde_json::json!({ + "cluster": resource, + "address": read_provider_gateway_address(site)?, + "transport": { + "mode": "mutual_tls", + "sni": format!("{site}.grid.internal") + } + })) + }) + .collect() +} - eprintln!(); - eprintln!(); - eprintln!( - "[SCENARIO {}] Prove equal round-robin across distinct provider gateways", - scenario() - ); - run_and_insert( - &mut results, - "provider_gateway_round_robin", - assert_provider_gateway_round_robin, - ); +/// Opt the run-owned provider-a `GatewayRef` into the operator-generated config. +#[expect( + clippy::too_many_lines, + reason = "this applies the opt-in only after reading and validating the run-owned gateway reference" +)] +fn enable_generated_consumer_config() -> Result<(), Box> { + let context = cluster_context("provider-a"); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + GRID_NETWORK_NAME, + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err("could not read provider-a GridNetwork before enabling consumerConfig".into()); + } + let network: serde_json::Value = serde_json::from_slice(&output.stdout)?; + let gateways = network + .pointer("/spec/gatewayRefs") + .and_then(serde_json::Value::as_array) + .ok_or("provider-a GridNetwork has no gatewayRefs array")?; + let index = gateways + .iter() + .position(|gateway| { + gateway.get("name").and_then(serde_json::Value::as_str) == Some("consumer-gateway") + && gateway.get("namespace").and_then(serde_json::Value::as_str) == Some(GRID_SYSTEM_NS) + }) + .ok_or("provider-a GridNetwork does not declare its run-owned consumer-gateway")?; + let gateway = gateways.get(index).ok_or("consumer-gateway index disappeared")?; + if gateway.get("consumerConfig").is_some() { + return Err("consumer-gateway already has consumerConfig; refusing to replace unexpected fixture state".into()); + } - results + let consumer_config = serde_json::json!({ + "enabled": true, + "enableProjectedCredentials": true, + "supportsProjectedCredentials": false, + "configMapName": "praxis-consumer-config", + "clusterEndpoints": generated_consumer_endpoints()?, + "tlsCertMountPath": "/etc/praxis/tls" + }); + let patch = serde_json::json!([{ + "op": "add", + "path": format!("/spec/gatewayRefs/{index}/consumerConfig"), + "value": consumer_config + }]); + let patched = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "patch", + "gridnetwork", + GRID_NETWORK_NAME, + "--type=json", + "--patch", + &patch.to_string(), + ]) + .output()?; + if !patched.status.success() { + return Err(format!( + "could not enable run-owned consumerConfig: {}", + safe_truncate_str(String::from_utf8_lossy(&patched.stderr).trim(), 800) + ) + .into()); + } + Ok(()) } -/// Send serial, unbound requests through one consumer gateway and verify that -/// the active no-metrics round-robin picker distributes them across the -/// distinct provider gateways. This is intentionally a request-path proof: -/// the request itself is the only source of the attribution counts. +/// Wait for a generation-current Rendered status and validate its live config. #[expect( clippy::too_many_lines, - clippy::unnecessary_wraps, - reason = "The assertion framework requires a fallible, named traffic proof boundary." + reason = "polling and validating the generated config is one bounded readiness gate" )] -fn assert_provider_gateway_round_robin() -> AssertionResult { - let start = Instant::now(); - let context = "kind-grid-provider-traffic-provider-a"; - let mut counts: BTreeMap = BTreeMap::new(); - let mut sequence = Vec::new(); - let mut failures = Vec::new(); - let mut overlay_changes = Vec::new(); +fn wait_for_generated_consumer_config() -> Result> { + const TIMEOUT: Duration = Duration::from_secs(180); + let context = cluster_context("provider-a"); + let deadline = Instant::now() + TIMEOUT; + let mut last_state = String::from("not observed"); + loop { + let network_output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + GRID_NETWORK_NAME, + "-o", + "json", + ]) + .output()?; + if network_output.status.success() { + let network: serde_json::Value = serde_json::from_slice(&network_output.stdout)?; + let generation = network + .pointer("/metadata/generation") + .and_then(serde_json::Value::as_i64); + let status = network + .pointer("/status/consumerConfigStatus") + .and_then(serde_json::Value::as_array); + let current = status.and_then(|items| { + items.iter().find(|item| { + item.get("gatewayName").and_then(serde_json::Value::as_str) == Some("consumer-gateway") + && item.get("namespace").and_then(serde_json::Value::as_str) == Some(GRID_SYSTEM_NS) + }) + }); + if let Some(current) = current { + let observed = current.get("observedGeneration").and_then(serde_json::Value::as_i64); + let phase = current + .get("phase") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + last_state = format!("phase={phase}, observed={observed:?}, generation={generation:?}"); + if phase == "Rendered" && observed == generation { + let config_output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + "praxis-consumer-config", + "-o", + "json", + ]) + .output()?; + if !config_output.status.success() { + return Err("GridNetwork reports consumer config Rendered but its ConfigMap is absent".into()); + } + let config_map: serde_json::Value = serde_json::from_slice(&config_output.stdout)?; + validate_generated_consumer_yaml(&config_map)?; + return Ok(serde_json::json!({ + "generation": generation, + "status": current, + "config_map": config_map, + })); + } + } + } + if Instant::now() >= deadline { + return Err(format!("timed out waiting for operator-generated consumer config: {last_state}").into()); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} - let readiness = match wait_for_round_robin_readiness() { - Ok(facts) => facts, - Err(error) => { - return Ok(proof_failure( - &format!("round-robin readiness gate failed: {error}"), - BTreeMap::from([(String::from("readiness_error"), serde_json::json!(error.to_string()))]), - start.elapsed(), - )); - }, - }; - let baseline_overlay = match read_cluster_overlay("provider-a") { - Ok(overlay) => overlay, - Err(error) => { - return Ok(proof_failure( - &format!("could not capture baseline overlay after readiness: {error}"), - BTreeMap::new(), - start.elapsed(), - )); - }, +/// Check that generated YAML is dynamic, scoped, and contains the complete +/// endpoint inventory rather than startup-only candidates. +#[expect( + clippy::too_many_lines, + reason = "the schema assertion checks the complete generated routing and endpoint contract" +)] +fn validate_generated_consumer_yaml(config_map: &serde_json::Value) -> Result<(), Box> { + let yaml = config_map + .pointer("/data/praxis.yaml") + .and_then(serde_json::Value::as_str) + .ok_or("praxis-consumer-config has no data.praxis.yaml")?; + let config: serde_yaml::Value = serde_yaml::from_str(yaml)?; + let filters = config + .get("filter_chains") + .and_then(serde_yaml::Value::as_sequence) + .and_then(|chains| chains.first()) + .and_then(|chain| chain.get("filters")) + .and_then(serde_yaml::Value::as_sequence) + .ok_or("generated consumer config has no filter chain")?; + let route = filters + .iter() + .find(|filter| filter.get("filter").and_then(serde_yaml::Value::as_str) == Some("intelligent_route")) + .ok_or("generated consumer config has no intelligent_route filter")?; + if route.get("overlay_file").and_then(serde_yaml::Value::as_str) != Some("/etc/praxis/routing/routing-overlay.json") + || route.get("candidates").is_some() + || route + .get("expected_overlay_scope") + .and_then(|scope| scope.get("network")) + .and_then(serde_yaml::Value::as_str) + != Some("grid-provider-traffic") + { + return Err("generated consumer config does not use the expected scoped dynamic overlay".into()); + } + let expected_provider_hops: BTreeSet<_> = PROVIDER_RESOURCES + .iter() + .map(|(_, cluster)| (*cluster).to_owned()) + .collect(); + let actual_provider_hops: BTreeSet<_> = route + .get("provider_hop_clusters") + .and_then(serde_yaml::Value::as_sequence) + .ok_or("generated consumer config has no provider_hop_clusters list")? + .iter() + .map(|cluster| { + cluster + .as_str() + .map(str::to_owned) + .ok_or("generated provider_hop_clusters contains a non-string entry") + }) + .collect::>()?; + if actual_provider_hops != expected_provider_hops { + return Err(format!( + "generated provider_hop_clusters mismatch: expected {expected_provider_hops:?}, got {actual_provider_hops:?}" + ) + .into()); + } + let credential_inject = filters + .iter() + .find(|filter| filter.get("filter").and_then(serde_yaml::Value::as_str) == Some("credential_inject")) + .ok_or("generated consumer config has no fail-closed credential_inject filter")?; + if credential_inject + .get("projected_credential_mount_base") + .and_then(serde_yaml::Value::as_str) + != Some("/run/secrets/grid-credentials") + { + return Err("generated consumer credential filter has no projected Secret mount fallback".into()); + } + if credential_inject + .get("credentials") + .and_then(serde_yaml::Value::as_sequence) + .is_none_or(|credentials| !credentials.is_empty()) + { + return Err("generated consumer credential filter must use an empty dynamic credential table".into()); + } + let cluster_count = filters + .iter() + .find(|filter| filter.get("filter").and_then(serde_yaml::Value::as_str) == Some("load_balancer")) + .and_then(|filter| filter.get("clusters")) + .and_then(serde_yaml::Value::as_sequence) + .map_or(0, Vec::len); + if cluster_count != PROVIDER_RESOURCES.len() { + return Err(format!( + "generated load balancer has {cluster_count} clusters; expected {}", + PROVIDER_RESOURCES.len() + ) + .into()); + } + Ok(()) +} + +/// Configure and exercise the chart-managed embedded `grid-gateway` consumer. +#[expect( + clippy::too_many_lines, + reason = "the qualification provisions one isolated gateway and verifies its baseline request" +)] +fn activate_embedded_serving_gateway() -> AssertionResult { + const TIMEOUT: Duration = Duration::from_secs(180); + let started = Instant::now(); + let image = std::env::var("GRID_XTASK_GRID_GATEWAY_IMAGE") + .unwrap_or_else(|_| "grid-gateway:provider-traffic-qualification".to_owned()); + require_local_image(&image)?; + let repository_root = grid_repository_root()?; + let operator_chart = repository_root.join("charts/grid-operator"); + + for cluster in CLUSTERS { + let kube_context = cluster_context(cluster); + let upgrade = Command::new("helm") + .args([ + "upgrade", + "grid-operator", + operator_chart.to_str().ok_or("Grid Operator chart path is not UTF-8")?, + "--kube-context", + &kube_context, + "--namespace", + GRID_SYSTEM_NS, + "--reuse-values", + "--set", + "signals.enabled=true", + "--wait", + "--timeout", + "180s", + ]) + .output()?; + if !upgrade.status.success() { + return Err(format!( + "could not enable the run-owned Grid signals listener in {cluster}: {}", + safe_truncate_str(String::from_utf8_lossy(&upgrade.stderr).trim(), 800) + ) + .into()); + } + } + + let endpoints = generated_consumer_endpoints()?; + for cluster in CLUSTERS { + patch_grid_network_for_serving(cluster, &endpoints)?; + } + for cluster in CLUSTERS { + restart_grid_operator(cluster)?; + } + + let all_clusters = BTreeSet::from([ + "vcr-provider-a-provider".to_owned(), + "vcr-provider-b-provider".to_owned(), + "vcr-provider-c-provider".to_owned(), + ]); + let initial_serving = wait_for_grid_serving_snapshot(&all_clusters, None, TIMEOUT)?; + require_serving_provider_hops(&initial_serving)?; + + if std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()) == "Never" { + let kind_name = format!( + "{}-{CONSUMER_SITE}", + RUN_CLUSTER_PREFIX + .get() + .ok_or("run-owned cluster prefix was not initialized")? + ); + crate::env::image_overrides::load_docker_image_into_kind(&image, &kind_name)?; + } + + apply_embedded_gateway_config(&cluster_context("provider-a"), &endpoints)?; + install_embedded_gateway(&repository_root, &image)?; + wait_for_deployment(GRID_SERVING_GATEWAY, GRID_SYSTEM_NS, &cluster_context("provider-a"))?; + let serving_log = wait_for_embedded_serving_log(&initial_serving, &cluster_context("provider-a"), 0, TIMEOUT)?; + let baseline = send_embedded_lifecycle_request(None)?; + if baseline.status != 200 || baseline.provider.is_none() { + return Err( + format!("embedded grid-gateway baseline did not reach an attributed provider: {baseline:?}").into(), + ); + } + let runtime_image = deployment_runtime_image_evidence(&cluster_context("provider-a"), GRID_SERVING_GATEWAY)?; + let facts = BTreeMap::from([ + ("gateway_image_reference".to_owned(), serde_json::json!(image)), + ( + "gateway_local_image_id".to_owned(), + serde_json::json!(docker_image_id(&image)?), + ), + ("gateway_runtime_image_ids".to_owned(), serde_json::json!(runtime_image)), + ( + "serving_config_initial_revision".to_owned(), + initial_serving + .get("digest") + .cloned() + .unwrap_or(serde_json::Value::Null), + ), + ( + "serving_config_initial_candidates".to_owned(), + initial_serving + .get("candidate_clusters") + .cloned() + .unwrap_or(serde_json::Value::Null), + ), + ("praxis_serving_log".to_owned(), serde_json::json!(serving_log)), + ("baseline_request".to_owned(), serde_json::json!(baseline)), + ]); + Ok(proof_success( + "chart-managed embedded grid-gateway loaded the Grid serving revision and routed an attributed request", + facts, + started.elapsed(), + )) +} + +/// Add the run-owned embedded gateway reference and select poll-backed serving config. +fn patch_grid_network_for_serving( + cluster: &str, + endpoints: &[serde_json::Value], +) -> Result<(), Box> { + let context = cluster_context(cluster); + let network = kubectl_get_json(&context, &format!("gridnetwork/{GRID_NETWORK_NAME}"))?; + let mut gateway_refs = network + .pointer("/spec/gatewayRefs") + .and_then(serde_json::Value::as_array) + .cloned() + .ok_or_else(|| format!("{cluster}: GridNetwork has no gatewayRefs array"))?; + if cluster == CONSUMER_SITE { + if gateway_refs + .iter() + .any(|gateway| gateway.get("name").and_then(serde_json::Value::as_str) == Some(GRID_SERVING_GATEWAY)) + { + return Err("run-owned GridNetwork unexpectedly already declares the embedded gateway".into()); + } + gateway_refs.push(embedded_serving_gateway_ref(endpoints)); + } + let patch = serde_json::json!({ + "spec": { + "signalTransport": { "mode": "poll" }, + "gatewayRefs": gateway_refs + } + }); + kubectl_patch_merge(&context, &format!("gridnetwork/{GRID_NETWORK_NAME}"), &patch)?; + Ok(()) +} + +/// Build the run-owned `GatewayRef` with the explicit mTLS provider-hop inventory. +fn embedded_serving_gateway_ref(endpoints: &[serde_json::Value]) -> serde_json::Value { + serde_json::json!({ + "name": GRID_SERVING_GATEWAY, + "namespace": GRID_SYSTEM_NS, + "localSiteName": CONSUMER_SITE, + "supportsEmptyOverlay": true, + "consumerConfig": { + "enabled": false, + "tlsCertMountPath": "/etc/praxis/tls", + "clusterEndpoints": endpoints + } + }) +} + +/// Require the embedded serving snapshot to carry the explicit provider-hop trust list. +fn require_serving_provider_hops(snapshot: &serde_json::Value) -> Result<(), Box> { + let expected: BTreeSet<_> = PROVIDER_RESOURCES + .iter() + .map(|(_, resource)| (*resource).to_owned()) + .collect(); + let actual: BTreeSet<_> = snapshot + .pointer("/serving_config/provider_hop_clusters") + .and_then(serde_json::Value::as_array) + .ok_or("embedded serving snapshot has no provider_hop_clusters allowlist")? + .iter() + .map(|cluster| { + cluster + .as_str() + .map(str::to_owned) + .ok_or("embedded provider-hop allowlist contains a non-string entry") + }) + .collect::>()?; + if actual != expected { + return Err( + format!("embedded serving provider-hop allowlist mismatch: expected {expected:?}, got {actual:?}").into(), + ); + } + Ok(()) +} + +/// Return public site labels used by the provider-attribution response header. +fn expected_provider_attributions() -> BTreeSet { + PROVIDER_RESOURCES.iter().map(|(site, _)| (*site).to_owned()).collect() +} + +/// Restart one run-owned operator after its `GridNetwork` declares poll mode. +fn restart_grid_operator(cluster: &str) -> Result<(), Box> { + let context = cluster_context(cluster); + let restart = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "rollout", + "restart", + "deployment/grid-operator", + ]) + .output()?; + if !restart.status.success() { + return Err(format!( + "{cluster}: could not restart the run-owned Grid operator for poll mode: {}", + safe_truncate_str(String::from_utf8_lossy(&restart.stderr).trim(), 500) + ) + .into()); + } + wait_for_deployment("grid-operator", GRID_SYSTEM_NS, &context)?; + Ok(()) +} + +/// Apply the embedded gateway's static listener/backend plumbing; Grid owns the changing candidates. +#[expect( + clippy::too_many_lines, + reason = "this renders listener plumbing and static endpoint inventory in one run-owned ConfigMap" +)] +fn apply_embedded_gateway_config( + context: &str, + endpoints: &[serde_json::Value], +) -> Result<(), Box> { + let clusters: Vec<_> = endpoints + .iter() + .map(|endpoint| { + let cluster = endpoint + .get("cluster") + .and_then(serde_json::Value::as_str) + .ok_or("generated endpoint has no cluster name")?; + let address = endpoint + .get("address") + .and_then(serde_json::Value::as_str) + .ok_or("generated endpoint has no address")?; + let sni = endpoint + .pointer("/transport/sni") + .and_then(serde_json::Value::as_str) + .ok_or("generated endpoint has no TLS server name")?; + Ok(serde_json::json!({ + "name": cluster, + "tls": { + "ca": { "ca_path": "/etc/praxis/tls/ca.crt" }, + "client_cert": { + "cert_path": "/etc/praxis/tls/tls.crt", + "key_path": "/etc/praxis/tls/tls.key" + }, + "sni": sni, + "verify": true + }, + "endpoints": [address] + })) + }) + .collect::>()?; + let config = serde_json::json!({ + "insecure_options": { + "allow_private_endpoints": true, + "allow_private_upstreams": true + }, + "listeners": [{ + "name": "proxy", + "address": "0.0.0.0:8080", + "filter_chains": ["main"] + }], + "filter_chains": [{ + "name": "main", + "filters": [ + { "filter": "model_to_header", "header": "X-Gateway-Model-Name" }, + { "filter": "grid_site_route", "model_header": "X-Gateway-Model-Name" }, + { "filter": "load_balancer", "clusters": clusters } + ] + }], + "admin": { "address": "127.0.0.1:9901" }, + "shutdown_timeout_secs": 5 + }); + let config_yaml = serde_yaml::to_string(&config)?; + let run_id = RUN_CLUSTER_PREFIX + .get() + .ok_or("run-owned cluster prefix was not initialized")?; + let config_map = serde_json::json!({ + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "grid-serving-gateway-config", + "namespace": GRID_SYSTEM_NS, + "labels": { "grid.praxis-proxy.io/run-id": run_id } + }, + "data": { "praxis.yaml": config_yaml } + }); + kubectl_apply_json(context, &config_map)?; + Ok(()) +} + +/// Install the chart-managed embedded consumer against the Grid serving `ConfigMap`. +#[expect( + clippy::too_many_lines, + reason = "Helm inputs and run-owned rollout verification form one bounded install operation" +)] +fn install_embedded_gateway(root: &Path, image: &str) -> Result<(), Box> { + let (repository, tag) = parse_image_ref(image); + let chart = root.join("charts/praxis-gateway"); + let chart = chart.to_str().ok_or("Praxis Gateway chart path is not UTF-8")?; + let context = cluster_context(CONSUMER_SITE); + let mut command = Command::new("helm"); + command.args([ + "upgrade", + "--install", + GRID_SERVING_GATEWAY, + chart, + "--kube-context", + &context, + "--namespace", + GRID_SYSTEM_NS, + "--wait", + "--timeout", + "180s", + ]); + for value in [ + format!("fullnameOverride={GRID_SERVING_GATEWAY}"), + format!("image.repository={repository}"), + format!("image.tag={tag}"), + "image.flavor=grid-gateway".to_owned(), + "image.pullPolicy=Never".to_owned(), + "config.existingConfigMap=grid-serving-gateway-config".to_owned(), + "service.type=ClusterIP".to_owned(), + format!("gridServing.network={GRID_NETWORK_NAME}"), + format!("gridServing.gatewayRef={GRID_SERVING_GATEWAY}"), + format!("gridServing.configMap={GRID_SERVING_CONFIGMAP}"), + format!("tls.existingSecret={CONSUMER_TLS_SECRET}"), + format!("tls.caSecret={CONSUMER_TLS_SECRET}"), + ] { + command.args(["--set-string", &value]); + } + command.args(["--set", "gridServing.enabled=true", "--set", "tls.enabled=true"]); + let output = command.output()?; + if !output.status.success() { + return Err(format!( + "could not install the run-owned embedded grid-gateway chart: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 1_000) + ) + .into()); + } + Ok(()) +} + +/// Wait for the operator's serving snapshot to match a candidate set and, optionally, a new digest. +fn wait_for_grid_serving_snapshot( + expected_clusters: &BTreeSet, + previous_digest: Option<&str>, + timeout: Duration, +) -> Result> { + let context = cluster_context(CONSUMER_SITE); + let deadline = Instant::now() + timeout; + let mut last = String::from("ConfigMap has not appeared"); + loop { + if let Ok(snapshot) = read_grid_serving_snapshot(&context) { + let digest = snapshot.get("digest").and_then(serde_json::Value::as_str); + let clusters = snapshot + .get("candidate_clusters") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect::>() + }) + .unwrap_or_default(); + let changed = previous_digest.is_none_or(|previous| digest != Some(previous)); + last = format!("digest={digest:?}, candidates={clusters:?}"); + if &clusters == expected_clusters && changed { + return Ok(snapshot); + } + } + if Instant::now() >= deadline { + return Err(format!("timed out waiting for embedded serving snapshot: {last}").into()); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} + +/// Read and verify the content-addressed operator serving snapshot. +fn read_grid_serving_snapshot(context: &str) -> Result> { + let config_map = kubectl_get_json(context, &format!("configmap/{GRID_SERVING_CONFIGMAP}"))?; + let text = config_map + .pointer("/data/serving-config.json") + .and_then(serde_json::Value::as_str) + .ok_or("Grid serving ConfigMap has no serving-config.json")?; + let digest = config_map + .pointer("/metadata/annotations/grid.praxis-proxy.io~1serving-digest") + .and_then(serde_json::Value::as_str) + .ok_or("Grid serving ConfigMap has no serving digest annotation")?; + let actual_digest = format!("{:x}", Sha256::digest(text.as_bytes())); + if digest != actual_digest { + return Err(format!("Grid serving digest annotation {digest} differs from content {actual_digest}").into()); + } + let parsed: serde_json::Value = serde_json::from_str(text)?; + let candidates = parsed + .get("candidates") + .and_then(serde_json::Value::as_array) + .ok_or("serving config has no candidates array")?; + let mut candidate_clusters: Vec<_> = candidates + .iter() + .filter_map(|candidate| candidate.get("cluster").and_then(serde_json::Value::as_str)) + .collect(); + candidate_clusters.sort_unstable(); + Ok(serde_json::json!({ + "digest": digest, + "candidate_count": candidates.len(), + "candidate_clusters": candidate_clusters, + "serving_config": parsed, + })) +} + +/// Wait for the embedded gateway process to log acceptance of one exact serving digest. +#[expect( + clippy::too_many_lines, + reason = "this bounded poll requires the exact digest and candidate count in runtime logs" +)] +fn wait_for_embedded_serving_log( + snapshot: &serde_json::Value, + context: &str, + prior_matching_observations: usize, + timeout: Duration, +) -> Result> { + let digest = snapshot + .get("digest") + .and_then(serde_json::Value::as_str) + .ok_or("serving snapshot evidence has no digest")?; + let count = snapshot + .get("candidate_count") + .and_then(serde_json::Value::as_u64) + .ok_or("serving snapshot evidence has no candidate count")?; + let deadline = Instant::now() + timeout; + let mut last_logs = String::new(); + loop { + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "logs", + &format!("deployment/{GRID_SERVING_GATEWAY}"), + "--all-containers", + "--tail=400", + ]) + .output()?; + if output.status.success() { + let raw_logs = String::from_utf8_lossy(&output.stdout); + let matches = embedded_serving_revision_lines(&raw_logs, digest, count); + if has_new_embedded_serving_revision_observation(&raw_logs, digest, count, prior_matching_observations) { + let matched_line = matches.last().ok_or("serving log match disappeared")?; + return Ok(safe_truncate_str(matched_line.trim(), 1_000)); + } + last_logs = strip_ansi_csi(&raw_logs); + } + if Instant::now() >= deadline { + return Err(format!( + "embedded gateway did not log serving digest {} with candidate_count={count}: {}", + safe_truncate_str(digest, 16), + safe_truncate_str(&last_logs, 1_000) + ) + .into()); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} + +/// Count existing exact serving-log observations so a repeated digest must be newly accepted. +fn embedded_serving_log_observation_count( + snapshot: &serde_json::Value, + context: &str, +) -> Result> { + let digest = snapshot + .get("digest") + .and_then(serde_json::Value::as_str) + .ok_or("serving snapshot evidence has no digest")?; + let count = snapshot + .get("candidate_count") + .and_then(serde_json::Value::as_u64) + .ok_or("serving snapshot evidence has no candidate count")?; + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "logs", + &format!("deployment/{GRID_SERVING_GATEWAY}"), + "--all-containers", + "--tail=400", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "could not read existing embedded serving logs: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 500) + ) + .into()); + } + Ok(embedded_serving_revision_lines(&String::from_utf8_lossy(&output.stdout), digest, count).len()) +} + +/// Send one request through the embedded `grid-gateway` Service and retain status plus provider attribution. +#[expect( + clippy::too_many_lines, + reason = "request headers, response status, and attribution are captured as one probe" +)] +fn send_embedded_lifecycle_request(session_id: Option<&str>) -> Result> { + let mut args = vec![ + "curl".to_owned(), + "--include".to_owned(), + "--silent".to_owned(), + "--show-error".to_owned(), + "--write-out".to_owned(), + format!("\n{CURL_HTTP_STATUS_MARKER}%{{http_code}}\n"), + "--header".to_owned(), + "Content-Type: application/json".to_owned(), + ]; + if let Some(session_id) = session_id { + args.push("--header".to_owned()); + args.push(format!("X-Session-Id: {session_id}")); + } + args.extend([ + "--header".to_owned(), + "x-ai-routing-candidate: caller-controlled-candidate".to_owned(), + "--header".to_owned(), + "x-ai-routing-request-id: caller-controlled-request".to_owned(), + "--header".to_owned(), + "x-ai-routing-revision: caller-controlled-revision".to_owned(), + "--data".to_owned(), + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"embedded no-route qualification"}],"max_tokens":8}"# + .to_owned(), + format!("http://{GRID_SERVING_GATEWAY}.{GRID_SYSTEM_NS}.svc.cluster.local:8080/v1/chat/completions"), + ]); + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let output = run_curl_probe(&cluster_context(CONSUMER_SITE), "embedded-withdrawal-probe", &refs)?; + if !output.status.success() { + return Err(format!( + "embedded gateway request command failed: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 500) + ) + .into()); + } + let status = curl_http_status(&output.stdout).map_err(|error| { + format!( + "embedded gateway response status unavailable: {error}; curl stderr: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 300) + ) + })?; + let provider = response_header(&output.stdout, "x-grid-provider-traffic-provider-gateway"); + Ok(LifecycleRequest { status, provider }) +} + +/// Get one run-cluster namespaced object as JSON. +fn kubectl_get_json(context: &str, resource: &str) -> Result> { + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "get", + resource, + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "could not read {resource}: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 500) + ) + .into()); + } + Ok(serde_json::from_slice(&output.stdout)?) +} + +/// Apply a merge patch to one run-cluster namespaced object. +fn kubectl_patch_merge( + context: &str, + resource: &str, + patch: &serde_json::Value, +) -> Result<(), Box> { + let patch = serde_json::to_string(patch)?; + let output = Command::new("kubectl") + .args([ + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "patch", + resource, + "--type=merge", + "--patch", + &patch, + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "could not patch run-owned {resource}: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 800) + ) + .into()); + } + Ok(()) +} + +/// Apply JSON through stdin, so generated `ConfigMap` data is not shell-interpreted. +fn kubectl_apply_json(context: &str, resource: &serde_json::Value) -> Result<(), Box> { + let mut child = Command::new("kubectl") + .args(["--context", context, "-n", GRID_SYSTEM_NS, "apply", "-f", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn()?; + let bytes = serde_json::to_vec(resource)?; + child + .stdin + .take() + .ok_or("kubectl stdin was not piped")? + .write_all(&bytes)?; + let output = child.wait_with_output()?; + if !output.status.success() { + return Err(format!( + "could not apply run-owned ConfigMap: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 800) + ) + .into()); + } + Ok(()) +} + +/// Return the local container-engine image ID for an exact image reference. +fn docker_image_id(image: &str) -> Result> { + let output = Command::new("docker") + .args(["image", "inspect", image, "--format", "{{.Id}}"]) + .output()?; + if !output.status.success() { + return Err(format!("could not inspect local image {image}").into()); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned()) +} + +/// Roll the run-owned consumer onto the exact operator-generated `ConfigMap` and +/// wait for the pod to load its dynamic overlay filter before traffic tests. +#[expect( + clippy::too_many_lines, + reason = "the helper patches only its run-owned Deployment and checks the exact loaded config" +)] +fn activate_generated_consumer_config() -> Result> { + enable_generated_consumer_config()?; + let evidence = wait_for_generated_consumer_config()?; + let context = cluster_context("provider-a"); + let patch = serde_json::json!({ + "spec": {"template": {"spec": {"volumes": [{ + "name": "config", + "configMap": {"name": "praxis-consumer-config"} + }]}}} + }); + let patched = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "patch", + "deployment", + "consumer-gateway", + "--type=strategic", + "--patch", + &patch.to_string(), + ]) + .output()?; + if !patched.status.success() { + return Err(format!( + "could not point consumer-gateway at generated config: {}", + safe_truncate_str(String::from_utf8_lossy(&patched.stderr).trim(), 800) + ) + .into()); + } + let rollout = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "rollout", + "status", + "deployment/consumer-gateway", + "--timeout=180s", + ]) + .output()?; + if !rollout.status.success() { + return Err(format!( + "consumer-gateway did not load generated config: {}", + safe_truncate_str(String::from_utf8_lossy(&rollout.stderr).trim(), 800) + ) + .into()); + } + let deployment = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "deployment", + "consumer-gateway", + "-o", + "json", + ]) + .output()?; + if !deployment.status.success() { + return Err("could not verify consumer-gateway config volume after rollout".into()); + } + let deployment: serde_json::Value = serde_json::from_slice(&deployment.stdout)?; + let uses_generated_config = deployment + .pointer("/spec/template/spec/volumes") + .and_then(serde_json::Value::as_array) + .is_some_and(|volumes| { + volumes.iter().any(|volume| { + volume.get("name").and_then(serde_json::Value::as_str) == Some("config") + && volume.pointer("/configMap/name").and_then(serde_json::Value::as_str) + == Some("praxis-consumer-config") + }) + }); + if !uses_generated_config { + return Err("consumer-gateway deployment is not mounted from praxis-consumer-config".into()); + } + let has_projected_credential_mount = deployment + .pointer("/spec/template/spec/containers") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter(|container| container.get("name").and_then(serde_json::Value::as_str) == Some("praxis")) + .flat_map(|container| { + container + .pointer("/volumeMounts") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + }) + .any(|mount| { + mount.get("name").and_then(serde_json::Value::as_str) == Some("credential-vcr-inference-credential") + && mount.get("mountPath").and_then(serde_json::Value::as_str) + == Some("/run/secrets/grid-credentials/grid-system/vcr-inference-credential") + && mount.get("readOnly").and_then(serde_json::Value::as_bool) == Some(true) + }); + if !has_projected_credential_mount { + return Err( + "consumer-gateway does not have the read-only provider Secret projection required by the generated filter" + .into(), + ); + } + let readiness = declare_projected_credentials_ready()?; + let ready_config = wait_for_generated_consumer_config()?; + let credential_overlay = enable_provider_a_credential_candidate()?; + let consumer_credential_request = verify_consumer_credential_request()?; + Ok(serde_json::json!({ + "operator_generated_config": evidence, + "readiness_attestation": readiness, + "ready_config_status": ready_config.get("status"), + "credential_overlay": credential_overlay, + "consumer_credential_request": consumer_credential_request, + "consumer_deployment_template": deployment.pointer("/spec/template/spec/volumes"), + "projected_credential_mount": "read-only vcr-inference-credential Secret projection verified (value not captured)", + "rollout": String::from_utf8_lossy(&rollout.stdout).trim(), + })) +} + +/// Prove the rolled-out consumer can route a credential-bearing provider-A +/// candidate without receiving the provider credential from the client. +fn verify_consumer_credential_request() -> Result> { + for attempt in 0..32 { + let session_id = format!("consumer-projected-credential-{attempt}"); + let request = send_lifecycle_request(Some(&session_id))?; + if request.provider.as_deref() == Some("provider-a") { + if request.status != 200 { + return Err(format!( + "consumer projected-credential request routed to provider-a but returned HTTP {}", + request.status + ) + .into()); + } + return Ok(serde_json::json!({ + "status": request.status, + "provider": request.provider, + "attempts_until_provider_a": attempt + 1, + "client_supplied_provider_secret": false, + "response_content_retained": false + })); + } + } + Err("no session-affinity probe selected credential-bearing provider-a within 32 attempts".into()) +} + +/// Attest the projected-credential capability only after the generated filter +/// and Secret volume have been confirmed in the rolled-out consumer Deployment. +#[expect( + clippy::too_many_lines, + reason = "one guarded read-patch-verify phase records a single readiness attestation" +)] +fn declare_projected_credentials_ready() -> Result> { + let context = cluster_context("provider-a"); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + GRID_NETWORK_NAME, + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err("could not read GridNetwork before acknowledging projected credentials".into()); + } + let network: serde_json::Value = serde_json::from_slice(&output.stdout)?; + let gateways = network + .pointer("/spec/gatewayRefs") + .and_then(serde_json::Value::as_array) + .ok_or("GridNetwork has no gatewayRefs array")?; + let index = gateways + .iter() + .position(|gateway| { + gateway.get("name").and_then(serde_json::Value::as_str) == Some("consumer-gateway") + && gateway.get("namespace").and_then(serde_json::Value::as_str) == Some(GRID_SYSTEM_NS) + }) + .ok_or("GridNetwork does not contain the run-owned consumer-gateway")?; + let gateway = gateways + .get(index) + .ok_or("GridNetwork consumer-gateway index is out of range")?; + let already_ready = gateway + .pointer("/consumerConfig/supportsProjectedCredentials") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if already_ready { + return Err("consumer projected-credential readiness was already set before this rollout".into()); + } + if gateway + .pointer("/consumerConfig/enableProjectedCredentials") + .and_then(serde_json::Value::as_bool) + != Some(true) + { + return Err("consumer projected credential filter was not enabled before rollout".into()); + } + let patch = serde_json::json!([{ + "op": "replace", + "path": format!("/spec/gatewayRefs/{index}/consumerConfig/supportsProjectedCredentials"), + "value": true + }]); + let patched = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "patch", + "gridnetwork", + GRID_NETWORK_NAME, + "--type=json", + "--patch", + &patch.to_string(), + ]) + .output()?; + if !patched.status.success() { + return Err(format!( + "could not acknowledge projected-credential rollout: {}", + safe_truncate_str(String::from_utf8_lossy(&patched.stderr).trim(), 800) + ) + .into()); + } + Ok(serde_json::json!({ + "enableProjectedCredentials": true, + "supportsProjectedCredentials": true, + "proof": "operator generated filter and read-only Secret mount verified after rollout" + })) +} + +/// Add one credential-bearing local candidate after the generated consumer +/// filter is rolled out, then wait until Grid publishes its Secret reference. +#[expect( + clippy::too_many_lines, + reason = "patch, bounded publication wait, and evidence form one ordered qualification phase" +)] +fn enable_provider_a_credential_candidate() -> Result> { + let previous = read_cluster_overlay("provider-a")?; + let context = cluster_context("provider-a"); + let patch = serde_json::json!({ + "spec": { + "auth": { + "strategy": "bearer_token", + "secretRef": { + "name": VCR_INFERENCE_CREDENTIAL, + "namespace": GRID_SYSTEM_NS, + "key": "token" + } + } + } + }); + let patched = Command::new("kubectl") + .args([ + "--context", + &context, + "patch", + "inferenceproviders.grid.praxis-proxy.io", + "vcr-provider-a-provider", + "--type=merge", + "--patch", + &patch.to_string(), + ]) + .output()?; + if !patched.status.success() { + return Err(format!( + "could not add the run-owned consumer credential reference: {}", + safe_truncate_str(String::from_utf8_lossy(&patched.stderr).trim(), 800) + ) + .into()); + } + + let deadline = Instant::now() + PROJECTED_CREDENTIAL_OVERLAY_TIMEOUT; + loop { + let last_state = match read_cluster_overlay("provider-a") { + Ok(overlay) => { + let candidate = overlay + .candidates + .iter() + .find(|candidate| candidate.site == "provider-a" && candidate.has_credential_reference); + if let Some(candidate) = candidate { + if overlay.semantic_revision != previous.semantic_revision { + return Ok(serde_json::json!({ + "previous_revision": previous.semantic_revision, + "credential_revision": overlay.semantic_revision, + "candidate_site": candidate.site, + "credential_reference_present": true, + "secret_value_captured": false + })); + } + "credential candidate observed without a new overlay revision".to_owned() + } else { + format!( + "revision={} has no credential-bearing provider-a candidate", + overlay.semantic_revision + ) + } + }, + Err(error) => error.to_string(), + }; + if Instant::now() >= deadline { + return Err(format!("timed out waiting for credential-bearing consumer overlay: {last_state}").into()); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} + +/// Run the original six proofs and append lifecycle coverage in full mode. +#[expect( + clippy::too_many_lines, + reason = "scenario order and the original six proofs remain explicit and auditable" +)] +fn run_provider_traffic_scenarios(mode: DemoMode, context: &ProviderTrafficContext) -> BTreeMap { + let mut results = BTreeMap::new(); + let mut scenario_num: usize = 0; + let mut scenario = || { + scenario_num += 1; + scenario_num + }; + + eprintln!(); + eprintln!("=== PROVIDER TRAFFIC SCENARIOS ==="); + eprintln!(); + + eprintln!("[SCENARIO {}] Verify three provider clusters are healthy", scenario()); + run_and_insert(&mut results, "cluster_health", assert_cluster_health); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify component deployment", scenario()); + run_and_insert(&mut results, "component_deployment", assert_component_deployment); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify SWIM convergence", scenario()); + run_and_insert(&mut results, "swim_convergence", assert_swim_convergence); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify site auto-discovery", scenario()); + run_and_insert(&mut results, "site_auto_discovery", assert_site_auto_discovery); + + eprintln!(); + eprintln!("[SCENARIO {}] Verify overlay acceptance", scenario()); + run_and_insert(&mut results, "overlay_acceptance", assert_overlay_acceptance); + + eprintln!(); + eprintln!(); + eprintln!( + "[SCENARIO {}] Prove equal round-robin across distinct provider gateways", + scenario() + ); + run_and_insert( + &mut results, + "provider_gateway_round_robin", + assert_provider_gateway_round_robin, + ); + + if provider_traffic_scenario_names(mode).contains(&"provider_withdrawal_lifecycle") { + eprintln!(); + eprintln!( + "[SCENARIO {}] Prove the embedded grid-gateway serving-config path is live and revision-aware", + scenario() + ); + let embedded = run_assertion("embedded_serving_gateway_convergence", || { + activate_embedded_serving_gateway() + }); + let embedded_ready = match embedded { + Ok(proof) => { + let ready = proof.success; + results.insert("embedded_serving_gateway_convergence".to_owned(), proof); + ready + }, + Err(error) => { + results.insert( + "embedded_serving_gateway_convergence".to_owned(), + proof_failure( + &format!("embedded serving gateway setup failed: {error}"), + BTreeMap::new(), + Duration::ZERO, + ), + ); + false + }, + }; + + eprintln!(); + eprintln!( + "[SCENARIO {}] Prove provider withdrawal, no-route serving, and restoration", + scenario() + ); + let proof = run_assertion("provider_withdrawal_lifecycle", || { + Ok(assert_provider_withdrawal_lifecycle(context, embedded_ready)) + }); + match proof { + Ok(proof) => { + results.insert("provider_withdrawal_lifecycle".to_owned(), proof); + }, + Err(error) => { + results.insert( + "provider_withdrawal_lifecycle".to_owned(), + proof_failure( + &format!("provider withdrawal lifecycle failed: {error}"), + BTreeMap::new(), + Duration::ZERO, + ), + ); + }, + } + } + + results +} + +/// Send serial, unbound requests through one consumer gateway and verify that +/// the active no-metrics round-robin picker distributes them across the +/// distinct provider gateways. This is intentionally a request-path proof: +/// the request itself is the only source of the attribution counts. +#[expect( + clippy::too_many_lines, + clippy::unnecessary_wraps, + reason = "The assertion framework requires a fallible, named traffic proof boundary." +)] +fn assert_provider_gateway_round_robin() -> AssertionResult { + let start = Instant::now(); + let context = cluster_context("provider-a"); + let mut counts: BTreeMap = BTreeMap::new(); + let mut sequence = Vec::new(); + let mut failures = Vec::new(); + let mut overlay_changes = Vec::new(); + + let readiness = match wait_for_round_robin_readiness() { + Ok(facts) => facts, + Err(error) => { + return Ok(proof_failure( + &format!("round-robin readiness gate failed: {error}"), + BTreeMap::from([(String::from("readiness_error"), serde_json::json!(error.to_string()))]), + start.elapsed(), + )); + }, + }; + let baseline_overlay = match read_cluster_overlay("provider-a") { + Ok(overlay) => overlay, + Err(error) => { + return Ok(proof_failure( + &format!("could not capture baseline overlay after readiness: {error}"), + BTreeMap::new(), + start.elapsed(), + )); + }, }; - for request_number in 1..=60 { - if let Ok(current_overlay) = read_cluster_overlay("provider-a") - && (current_overlay.resource_version != baseline_overlay.resource_version - || current_overlay.semantic_revision != baseline_overlay.semantic_revision) + for request_number in 1..=60 { + if let Ok(current_overlay) = read_cluster_overlay("provider-a") + && (current_overlay.resource_version != baseline_overlay.resource_version + || current_overlay.semantic_revision != baseline_overlay.semantic_revision) + { + overlay_changes.push(serde_json::json!({ + "request": request_number, + "resource_version": current_overlay.resource_version, + "semantic_revision": current_overlay.semantic_revision, + })); + } + let request_label = format!("provider-traffic-rr-{request_number:03}"); + let output = run_curl_probe( + &context, + &format!("rr-{request_number}"), + &[ + "curl", + "--fail-with-body", + "--include", + "--silent", + "--show-error", + "--header", + "Content-Type: application/json", + "--header", + "Authorization: Bearer consumer-token", + "--data", + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"provider traffic proof"}],"max_tokens":8}"#, + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", + ], + ); + + match output { + Ok(output) if output.status.success() => { + let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") + .or_else(|| response_header(&output.stdout, "x-grid-provider-traffic-provider-gateway")) + .or_else(|| response_header(&output.stdout, "x-grid-provider-gateway")); + if let Some(provider) = provider { + *counts.entry(provider.clone()).or_default() += 1; + sequence.push(provider); + } else { + failures.push(format!("{request_label}: provider attribution header missing")); + } + }, + Ok(output) => failures.push(format!( + "{request_label}: HTTP probe failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )), + Err(error) => failures.push(format!("{request_label}: probe execution failed: {error}")), + } + } + + let canonical = ["provider-a", "provider-b", "provider-c"]; + let exact_counts = canonical + .iter() + .all(|name| counts.get(*name).copied().unwrap_or(0) == 20); + let cycle_start = sequence + .first() + .and_then(|first| canonical.iter().position(|name| *name == first)); + let repeating_cycle = sequence.len() == 60 + && cycle_start.is_some_and(|start_index| { + sequence.iter().enumerate().all(|(index, provider)| { + canonical + .get((start_index + index) % canonical.len()) + .is_some_and(|expected| *expected == provider.as_str()) + }) + }); + let balanced = exact_counts && repeating_cycle && failures.is_empty(); + + let mut facts = BTreeMap::new(); + facts.insert("request_count".to_owned(), serde_json::json!(sequence.len())); + facts.insert("provider_counts".to_owned(), serde_json::json!(counts)); + facts.insert("ordered_provider_sequence".to_owned(), serde_json::json!(sequence)); + facts.insert("cycle_start_provider".to_owned(), serde_json::json!(sequence.first())); + facts.insert("exact_20_each".to_owned(), serde_json::json!(exact_counts)); + facts.insert( + "repeating_three_provider_cycle".to_owned(), + serde_json::json!(repeating_cycle), + ); + facts.insert("failures".to_owned(), serde_json::json!(failures)); + facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); + facts.insert("scoring_strategy".to_owned(), serde_json::json!("noMetrics")); + facts.insert("readiness".to_owned(), serde_json::json!(readiness)); + facts.insert( + "baseline_resource_version".to_owned(), + serde_json::json!(baseline_overlay.resource_version), + ); + facts.insert( + "baseline_semantic_revision".to_owned(), + serde_json::json!(baseline_overlay.semantic_revision), + ); + let overlay_stable = overlay_changes.is_empty(); + facts.insert( + "overlay_changes_during_requests".to_owned(), + serde_json::json!(overlay_changes), + ); + + if balanced && overlay_stable { + Ok(proof_success( + "60 serial requests distributed evenly across three distinct provider gateways", + facts, + start.elapsed(), + )) + } else { + Ok(proof_failure( + &format!( + "provider gateway distribution or overlay stability failed: counts={counts:?}, overlay_changes={overlay_changes:?}" + ), + facts, + start.elapsed(), + )) + } +} + +/// Snapshot of the selector that the lifecycle test temporarily changes. +#[derive(Clone, Debug, Serialize)] +struct ProviderSelectorState { + /// Provider site identity. + site: String, + /// Run-owned `InferenceProvider` name. + resource: String, + /// Original selector value restored in all exit paths. + selector_value: String, + /// Object UID captured before mutation. + uid: String, + /// Resource version captured before mutation. + resource_version: String, +} + +/// Capture the selector contract from one topology-owned `InferenceProvider`. +#[expect( + clippy::too_many_lines, + reason = "the helper validates identity and preserves every field needed for safe restoration" +)] +fn read_provider_selector(site: &str, resource: &str) -> Result> { + let context = cluster_context(site); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "inferenceprovider", + resource, + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err(format!("{site}: could not read InferenceProvider/{resource}").into()); + } + let provider: serde_json::Value = serde_json::from_slice(&output.stdout)?; + let selector_value = provider + .get("spec") + .and_then(|spec| spec.get("siteSelector")) + .and_then(|selector| selector.get("matchLabels")) + .and_then(|labels| labels.get(PROVIDER_SITE_LABEL)) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| format!("{site}: InferenceProvider/{resource} has no expected site selector"))? + .to_owned(); + if selector_value != site { + return Err(format!("{site}: refusing to mutate unexpected site selector {selector_value:?}").into()); + } + Ok(ProviderSelectorState { + site: site.to_owned(), + resource: resource.to_owned(), + selector_value, + uid: provider + .pointer("/metadata/uid") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown") + .to_owned(), + resource_version: provider + .pointer("/metadata/resourceVersion") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown") + .to_owned(), + }) +} + +/// Change one topology-owned provider selector without touching its workload. +#[expect( + clippy::too_many_lines, + reason = "selector mutation validates its target and applies only the narrowly owned label change" +)] +fn patch_provider_selector( + state: &ProviderSelectorState, + selector_value: &str, +) -> Result<(), Box> { + let context = cluster_context(&state.site); + let mut patch = serde_json::json!({"spec": {"siteSelector": {"matchLabels": {}}}}); + let labels = patch + .pointer_mut("/spec/siteSelector/matchLabels") + .and_then(serde_json::Value::as_object_mut) + .ok_or("failed to construct selector patch")?; + labels.insert(PROVIDER_SITE_LABEL.to_owned(), serde_json::json!(selector_value)); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "patch", + "inferenceprovider", + &state.resource, + "--type=merge", + "--patch", + &patch.to_string(), + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "{}: patching InferenceProvider/{} selector failed: {}", + state.site, + state.resource, + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 500) + ) + .into()); + } + Ok(()) +} + +/// Wait for a stable overlay containing exactly the requested clusters. +fn wait_for_overlay_clusters( + expected: &BTreeSet, + previous_revision: Option<&str>, +) -> Result> { + const TIMEOUT: Duration = Duration::from_secs(180); + let deadline = Instant::now() + TIMEOUT; + let mut stable_revision: Option = None; + loop { + let overlay = read_cluster_overlay("provider-a")?; + let clusters: BTreeSet = overlay + .candidates + .iter() + .map(|candidate| candidate.cluster.clone()) + .collect(); + let changed = previous_revision.is_none_or(|previous| overlay.semantic_revision != previous); + if changed && &clusters == expected { + if stable_revision.as_deref() == Some(overlay.semantic_revision.as_str()) { + return Ok(overlay); + } + stable_revision = Some(overlay.semantic_revision.clone()); + } else { + stable_revision = None; + } + if Instant::now() >= deadline { + return Err(format!( + "timed out waiting for overlay clusters {expected:?}; last clusters={clusters:?}, revision={}", + safe_truncate_str(&overlay.semantic_revision, 16) + ) + .into()); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} + +/// Issue a request and retain only status plus provider attribution. +#[expect( + clippy::too_many_lines, + reason = "the lifecycle probe constructs a bounded request and parses only status/attribution" +)] +fn send_lifecycle_request(session_id: Option<&str>) -> Result> { + let mut args = vec![ + "curl".to_owned(), + "--include".to_owned(), + "--silent".to_owned(), + "--show-error".to_owned(), + "--write-out".to_owned(), + format!("\n{CURL_HTTP_STATUS_MARKER}%{{http_code}}\n"), + "--header".to_owned(), + "Content-Type: application/json".to_owned(), + "--header".to_owned(), + "Authorization: Bearer consumer-token".to_owned(), + ]; + if let Some(session_id) = session_id { + args.push("--header".to_owned()); + args.push(format!("X-Session-Id: {session_id}")); + } + args.extend([ + "--data".to_owned(), + r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"no-route qualification"}],"max_tokens":8}"# + .to_owned(), + "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions".to_owned(), + ]); + let refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let output = run_curl_probe(&cluster_context("provider-a"), "withdrawal-probe", &refs)?; + if !output.status.success() { + return Err(format!( + "consumer request command failed: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 500) + ) + .into()); + } + let status = curl_http_status(&output.stdout).map_err(|error| { + format!( + "consumer response status unavailable: {error}; curl stderr: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 300) + ) + })?; + let provider = response_header(&output.stdout, "x-grid-provider-traffic-provider-gateway"); + Ok(LifecycleRequest { status, provider }) +} + +/// Read the deterministic backend's successful inference count. +#[expect( + clippy::too_many_lines, + reason = "the scoped port-forward lifecycle, bounded HTTP read, and metric parsing form one observation contract" +)] +fn read_backend_success_count(site: &str) -> Result> { + let context = cluster_context(site); + let service = format!("service/vcr-inference-{site}"); + let local_port = find_local_tcp_port()?; + let mapping = format!("{local_port}:8000"); + let child = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "port-forward", + &service, + &mapping, + "--address", + "127.0.0.1", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + let _port_forward = BackendMetricsPortForward { child: Some(child) }; + if !wait_for_local_tcp_port(local_port, Duration::from_secs(10)) { + return Err(format!("{site}: simulator metrics port-forward did not become ready").into()); + } + + let url = format!("http://127.0.0.1:{local_port}/metrics"); + let output = Command::new("curl") + .args([ + "--fail", + "--silent", + "--show-error", + "--connect-timeout", + "2", + "--max-time", + "5", + &url, + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "{site}: simulator /metrics probe through the run-scoped port-forward failed: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 300) + ) + .into()); + } + let metrics = String::from_utf8_lossy(&output.stdout); + let mut found = false; + let mut total = 0_u64; + for line in metrics + .lines() + .filter(|line| line.starts_with("vllm:request_success_total")) + { + let value = line + .split_whitespace() + .last() + .ok_or_else(|| format!("{site}: malformed simulator success metric"))? + .parse::()?; + total = total.checked_add(value).ok_or("simulator metric sum overflow")?; + found = true; + } + if !found { + return Err(format!("{site}: simulator did not expose vllm:request_success_total").into()); + } + Ok(total) +} + +/// Reserve and release an ephemeral loopback port for one metrics observation. +fn find_local_tcp_port() -> Result> { + let listener = TcpListener::bind("127.0.0.1:0")?; + Ok(listener.local_addr()?.port()) +} + +/// Wait until the run-scoped `kubectl port-forward` accepts a loopback connection. +fn wait_for_local_tcp_port(port: u16, timeout: Duration) -> bool { + let deadline = Instant::now() + timeout; + loop { + if TcpStream::connect(("127.0.0.1", port)).is_ok() { + return true; + } + if Instant::now() >= deadline { + return false; + } + std::thread::park_timeout(Duration::from_millis(100)); + } +} + +/// Write incremental no-route evidence so failed runs retain their last phase. +fn write_lifecycle_evidence( + context: &ProviderTrafficContext, + facts: &BTreeMap, +) -> Result<(), Box> { + fs::write( + context.evidence_dir.join("no-route-lifecycle.json"), + serde_json::to_vec_pretty(facts)?, + )?; + Ok(()) +} + +/// Prove partial fallback, authoritative empty serving, backend non-contact, +/// session invalidation, and restoration on the running consumer gateway. +#[expect( + clippy::too_many_lines, + reason = "the assertion documents ordered mutations and unconditional restoration" +)] +#[expect( + clippy::large_stack_frames, + reason = "bounded lifecycle evidence and cleanup state live for one synchronous qualification" +)] +fn assert_provider_withdrawal_lifecycle( + context: &ProviderTrafficContext, + embedded_gateway_enabled: bool, +) -> ProofResult { + let start = Instant::now(); + let mut facts = BTreeMap::new(); + let mut original_selectors = Vec::new(); + let mut embedded_restore_prior_log_observations = 0; + let body = (|| -> Result<(), Box> { + for (site, resource) in PROVIDER_RESOURCES { + wait_for_deployment("provider-gateway", GRID_SYSTEM_NS, &cluster_context(site))?; + wait_for_deployment(&format!("vcr-inference-{site}"), GRID_SYSTEM_NS, &cluster_context(site))?; + original_selectors.push(read_provider_selector(site, resource)?); + } + facts.insert( + "provider_selectors_before".to_owned(), + serde_json::json!(original_selectors), + ); + write_lifecycle_evidence(context, &facts)?; + + let baseline = read_cluster_overlay("provider-a")?; + let all_clusters: BTreeSet = [ + "vcr-provider-a-provider", + "vcr-provider-b-provider", + "vcr-provider-c-provider", + ] + .into_iter() + .map(str::to_owned) + .collect(); + if baseline + .candidates + .iter() + .map(|candidate| candidate.cluster.clone()) + .collect::>() + != all_clusters { - overlay_changes.push(serde_json::json!({ - "request": request_number, - "resource_version": current_overlay.resource_version, - "semantic_revision": current_overlay.semantic_revision, - })); + return Err("lifecycle did not start from exactly providers A, B, and C".into()); } - let request_label = format!("provider-traffic-rr-{request_number:03}"); - let output = run_curl_probe( - context, - &format!("rr-{request_number}"), - &[ - "curl", - "--fail-with-body", - "--include", - "--silent", - "--show-error", - "--header", - "Content-Type: application/json", - "--header", - "Authorization: Bearer consumer-token", - "--data", - r#"{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"provider traffic proof"}],"max_tokens":8}"#, - "http://consumer-gateway.grid-system.svc.cluster.local:8080/v1/chat/completions", - ], + wait_for_consumer_gateway_revision(&baseline.semantic_revision)?; + let mut embedded_serving_digest = if embedded_gateway_enabled { + let embedded_baseline = read_grid_serving_snapshot(&cluster_context(CONSUMER_SITE))?; + embedded_restore_prior_log_observations = + embedded_serving_log_observation_count(&embedded_baseline, &cluster_context(CONSUMER_SITE))?; + if embedded_restore_prior_log_observations == 0 { + return Err("embedded gateway has not logged acceptance of its baseline serving revision".into()); + } + facts.insert("embedded_baseline_serving_config".to_owned(), embedded_baseline.clone()); + facts.insert( + "embedded_baseline_log_observation_count".to_owned(), + serde_json::json!(embedded_restore_prior_log_observations), + ); + Some( + embedded_baseline + .get("digest") + .and_then(serde_json::Value::as_str) + .ok_or("embedded baseline has no serving digest")? + .to_owned(), + ) + } else { + None + }; + facts.insert( + "baseline_revision".to_owned(), + serde_json::json!(baseline.semantic_revision), ); + facts.insert("baseline_candidates".to_owned(), serde_json::json!(baseline.candidates)); + write_lifecycle_evidence(context, &facts)?; + + let mut bound_session = None; + for attempt in 0..9 { + let session_id = format!("grid258-bound-to-a-{attempt}"); + let response = send_lifecycle_request(Some(&session_id))?; + if response.status != 200 { + return Err(format!("baseline affinity request returned HTTP {}", response.status).into()); + } + if response.provider.as_deref() == Some("provider-a") { + bound_session = Some(session_id); + break; + } + } + let bound_session = bound_session.ok_or("could not bind a request session to provider A")?; + facts.insert("provider_a_bound_session".to_owned(), serde_json::json!(bound_session)); + write_lifecycle_evidence(context, &facts)?; - match output { - Ok(output) if output.status.success() => { - let provider = response_header(&output.stdout, "x-grid-combined-provider-gateway") - .or_else(|| response_header(&output.stdout, "x-grid-provider-traffic-provider-gateway")) - .or_else(|| response_header(&output.stdout, "x-grid-provider-gateway")); - if let Some(provider) = provider { - *counts.entry(provider.clone()).or_default() += 1; - sequence.push(provider); - } else { - failures.push(format!("{request_label}: provider attribution header missing")); + let a = original_selectors + .iter() + .find(|selector| selector.site == "provider-a") + .ok_or("missing captured provider A selector")?; + patch_provider_selector(a, "grid258-withdrawn-provider-a")?; + facts.insert( + "withdrawn_provider_a_selector".to_owned(), + serde_json::json!("grid258-withdrawn-provider-a"), + ); + write_lifecycle_evidence(context, &facts)?; + let fallback_set = BTreeSet::from([ + "vcr-provider-b-provider".to_owned(), + "vcr-provider-c-provider".to_owned(), + ]); + let fallback = wait_for_overlay_clusters(&fallback_set, Some(&baseline.semantic_revision))?; + wait_for_consumer_gateway_revision(&fallback.semantic_revision)?; + facts.insert( + "fallback_revision".to_owned(), + serde_json::json!(fallback.semantic_revision), + ); + facts.insert("fallback_candidates".to_owned(), serde_json::json!(fallback.candidates)); + write_lifecycle_evidence(context, &facts)?; + + if embedded_gateway_enabled { + let expected = BTreeSet::from([ + "vcr-provider-b-provider".to_owned(), + "vcr-provider-c-provider".to_owned(), + ]); + let serving = wait_for_grid_serving_snapshot( + &expected, + embedded_serving_digest.as_deref(), + Duration::from_secs(180), + )?; + let log = + wait_for_embedded_serving_log(&serving, &cluster_context(CONSUMER_SITE), 0, Duration::from_secs(90))?; + for attempt in 0..6 { + let response = send_embedded_lifecycle_request(Some(&format!("grid258-embedded-fallback-{attempt}")))?; + if response.status != 200 || response.provider.as_deref() == Some("provider-a") { + return Err(format!( + "embedded serving-config fallback reached a hard-withdrawn provider or failed: {response:?}" + ) + .into()); + } + } + facts.insert("embedded_fallback_serving_config".to_owned(), serving.clone()); + facts.insert("embedded_fallback_serving_log".to_owned(), serde_json::json!(log)); + facts.insert("embedded_fallback_requests".to_owned(), serde_json::json!(6)); + embedded_serving_digest = Some( + serving + .get("digest") + .and_then(serde_json::Value::as_str) + .ok_or("embedded fallback has no serving digest")? + .to_owned(), + ); + write_lifecycle_evidence(context, &facts)?; + } + + let mut fallback_providers = BTreeSet::new(); + for _ in 0..6 { + let response = send_lifecycle_request(None)?; + if response.status != 200 { + return Err(format!("partial-withdrawal request returned HTTP {}", response.status).into()); + } + let provider = response + .provider + .ok_or("fallback request had no provider attribution")?; + if provider == "provider-a" { + return Err("hard-withdrawn provider A still served a new request".into()); + } + fallback_providers.insert(provider); + } + if fallback_providers != BTreeSet::from(["provider-b".to_owned(), "provider-c".to_owned()]) { + return Err(format!( + "partial withdrawal did not route across both eligible fallbacks: {fallback_providers:?}" + ) + .into()); + } + facts.insert("fallback_attribution".to_owned(), serde_json::json!(fallback_providers)); + write_lifecycle_evidence(context, &facts)?; + + for state in original_selectors.iter().filter(|state| state.site != "provider-a") { + patch_provider_selector(state, &format!("grid258-withdrawn-{}", state.site))?; + } + facts.insert("withdrawn_provider_b_c_selectors".to_owned(), serde_json::json!(true)); + write_lifecycle_evidence(context, &facts)?; + let empty = wait_for_overlay_clusters(&BTreeSet::new(), Some(&fallback.semantic_revision))?; + wait_for_consumer_gateway_revision(&empty.semantic_revision)?; + facts.insert("empty_revision".to_owned(), serde_json::json!(empty.semantic_revision)); + facts.insert( + "empty_candidate_count".to_owned(), + serde_json::json!(empty.candidates.len()), + ); + facts.insert( + "praxis_empty_serving_revision".to_owned(), + serde_json::json!(empty.semantic_revision), + ); + if embedded_gateway_enabled { + let serving = wait_for_grid_serving_snapshot( + &BTreeSet::new(), + embedded_serving_digest.as_deref(), + Duration::from_secs(180), + )?; + require_serving_provider_hops(&serving)?; + let log = + wait_for_embedded_serving_log(&serving, &cluster_context(CONSUMER_SITE), 0, Duration::from_secs(90))?; + facts.insert("embedded_empty_serving_config".to_owned(), serving); + facts.insert("embedded_empty_serving_log".to_owned(), serde_json::json!(log)); + embedded_serving_digest = Some( + read_grid_serving_snapshot(&cluster_context(CONSUMER_SITE))? + .get("digest") + .and_then(serde_json::Value::as_str) + .ok_or("embedded empty snapshot has no serving digest")? + .to_owned(), + ); + } + write_lifecycle_evidence(context, &facts)?; + + for (site, _) in PROVIDER_RESOURCES { + wait_for_deployment("provider-gateway", GRID_SYSTEM_NS, &cluster_context(site))?; + wait_for_deployment(&format!("vcr-inference-{site}"), GRID_SYSTEM_NS, &cluster_context(site))?; + } + let metrics_before: BTreeMap = PROVIDER_RESOURCES + .iter() + .map(|(site, _)| Ok(((*site).to_owned(), read_backend_success_count(site)?))) + .collect::>>()?; + facts.insert( + "backend_success_total_before_no_route".to_owned(), + serde_json::json!(metrics_before), + ); + write_lifecycle_evidence(context, &facts)?; + + let mut no_route_results = Vec::new(); + for attempt in 0..3 { + no_route_results.push(send_lifecycle_request(Some(&format!( + "grid258-new-no-route-{attempt}" + )))?); + } + no_route_results.push(send_lifecycle_request(Some(&bound_session))?); + if no_route_results + .iter() + .any(|response| response.status != 404 || response.provider.is_some()) + { + return Err(format!("no-route requests were not all unattributed HTTP 404: {no_route_results:?}").into()); + } + facts.insert("no_route_requests".to_owned(), serde_json::json!(no_route_results)); + if embedded_gateway_enabled { + let embedded_no_route: Vec<_> = (0..3) + .map(|attempt| send_embedded_lifecycle_request(Some(&format!("grid258-embedded-no-route-{attempt}")))) + .collect::>()?; + if embedded_no_route + .iter() + .any(|response| response.status != 404 || response.provider.is_some()) + { + return Err(format!( + "embedded gateway continued to route after serving empty revision: {embedded_no_route:?}" + ) + .into()); + } + let bound_probe = send_embedded_lifecycle_request(Some(&bound_session))?; + if bound_probe.status != 404 || bound_probe.provider.is_some() { + return Err(format!( + "embedded gateway routed a request with a session header despite an empty candidate snapshot: {bound_probe:?}" + ) + .into()); + } + facts.insert( + "embedded_no_route_requests".to_owned(), + serde_json::json!(embedded_no_route), + ); + facts.insert( + "embedded_session_header_no_route_probe".to_owned(), + serde_json::json!(bound_probe), + ); + } + + let metrics_after: BTreeMap = PROVIDER_RESOURCES + .iter() + .map(|(site, _)| Ok(((*site).to_owned(), read_backend_success_count(site)?))) + .collect::>>()?; + if metrics_before != metrics_after { + return Err(format!("backend request counters changed while no route was serving: before={metrics_before:?}, after={metrics_after:?}").into()); + } + facts.insert( + "backend_success_total_after_no_route".to_owned(), + serde_json::json!(metrics_after), + ); + facts.insert("backend_non_contact_proven".to_owned(), serde_json::json!(true)); + if embedded_gateway_enabled { + facts.insert( + "embedded_backend_non_contact_proven".to_owned(), + serde_json::json!(true), + ); + } + write_lifecycle_evidence(context, &facts)?; + + for state in &original_selectors { + patch_provider_selector(state, &state.selector_value)?; + } + let restored = wait_for_overlay_clusters(&all_clusters, Some(&empty.semantic_revision))?; + wait_for_consumer_gateway_revision(&restored.semantic_revision)?; + facts.insert( + "restored_revision".to_owned(), + serde_json::json!(restored.semantic_revision), + ); + facts.insert("restored_candidates".to_owned(), serde_json::json!(restored.candidates)); + if embedded_gateway_enabled { + let serving = wait_for_grid_serving_snapshot( + &all_clusters, + embedded_serving_digest.as_deref(), + Duration::from_secs(180), + )?; + let log = wait_for_embedded_serving_log( + &serving, + &cluster_context(CONSUMER_SITE), + embedded_restore_prior_log_observations, + Duration::from_secs(90), + )?; + facts.insert("embedded_restored_serving_config".to_owned(), serving); + facts.insert("embedded_restored_serving_log".to_owned(), serde_json::json!(log)); + } + let response = send_lifecycle_request(None)?; + if response.status != 200 + || !expected_provider_attributions().contains(response.provider.as_deref().unwrap_or_default()) + { + return Err(format!("restored provider request was not attributed to A/B/C: {response:?}").into()); + } + facts.insert("restored_request".to_owned(), serde_json::json!(response)); + if embedded_gateway_enabled { + let embedded_response = send_embedded_lifecycle_request(None)?; + if embedded_response.status != 200 + || !expected_provider_attributions().contains(embedded_response.provider.as_deref().unwrap_or_default()) + { + return Err( + format!("embedded gateway did not restore attributed traffic: {embedded_response:?}").into(), + ); + } + facts.insert( + "embedded_restored_request".to_owned(), + serde_json::json!(embedded_response), + ); + } + facts.insert("provider_workloads_remained_ready".to_owned(), serde_json::json!(true)); + write_lifecycle_evidence(context, &facts)?; + Ok(()) + })(); + + let mut restoration_error = None; + if !original_selectors.is_empty() { + let before_restore = read_cluster_overlay("provider-a").ok(); + for state in &original_selectors { + if let Err(error) = patch_provider_selector(state, &state.selector_value) { + restoration_error = Some(format!("{}: {error}", state.site)); + } + } + if restoration_error.is_none() { + let all_clusters: BTreeSet = PROVIDER_RESOURCES + .iter() + .map(|(_, resource)| (*resource).to_owned()) + .collect(); + let current_clusters = before_restore.as_ref().map(|overlay| { + overlay + .candidates + .iter() + .map(|candidate| candidate.cluster.clone()) + .collect::>() + }); + if current_clusters.as_ref() != Some(&all_clusters) { + let previous_revision = before_restore + .as_ref() + .map(|overlay| overlay.semantic_revision.as_str()); + if let Err(error) = wait_for_overlay_clusters(&all_clusters, previous_revision) { + restoration_error = Some(error.to_string()); } - }, - Ok(output) => failures.push(format!( - "{request_label}: HTTP probe failed: {}", - String::from_utf8_lossy(&output.stderr).trim() - )), - Err(error) => failures.push(format!("{request_label}: probe execution failed: {error}")), + } } } - - let canonical = ["provider-a", "provider-b", "provider-c"]; - let exact_counts = canonical - .iter() - .all(|name| counts.get(*name).copied().unwrap_or(0) == 20); - let cycle_start = sequence - .first() - .and_then(|first| canonical.iter().position(|name| *name == first)); - let repeating_cycle = sequence.len() == 60 - && cycle_start.is_some_and(|start_index| { - sequence.iter().enumerate().all(|(index, provider)| { - canonical - .get((start_index + index) % canonical.len()) - .is_some_and(|expected| *expected == provider.as_str()) - }) - }); - let balanced = exact_counts && repeating_cycle && failures.is_empty(); - - let mut facts = BTreeMap::new(); - facts.insert("request_count".to_owned(), serde_json::json!(sequence.len())); - facts.insert("provider_counts".to_owned(), serde_json::json!(counts)); - facts.insert("ordered_provider_sequence".to_owned(), serde_json::json!(sequence)); - facts.insert("cycle_start_provider".to_owned(), serde_json::json!(sequence.first())); - facts.insert("exact_20_each".to_owned(), serde_json::json!(exact_counts)); facts.insert( - "repeating_three_provider_cycle".to_owned(), - serde_json::json!(repeating_cycle), - ); - facts.insert("failures".to_owned(), serde_json::json!(failures)); - facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); - facts.insert("scoring_strategy".to_owned(), serde_json::json!("noMetrics")); - facts.insert("readiness".to_owned(), serde_json::json!(readiness)); - facts.insert( - "baseline_resource_version".to_owned(), - serde_json::json!(baseline_overlay.resource_version), - ); - facts.insert( - "baseline_semantic_revision".to_owned(), - serde_json::json!(baseline_overlay.semantic_revision), + "selector_restoration_error".to_owned(), + serde_json::json!(restoration_error), ); - let overlay_stable = overlay_changes.is_empty(); facts.insert( - "overlay_changes_during_requests".to_owned(), - serde_json::json!(overlay_changes), + "lifecycle_error".to_owned(), + serde_json::json!(body.as_ref().err().map(ToString::to_string)), ); - - if balanced && overlay_stable { - Ok(proof_success( - "60 serial requests distributed evenly across three distinct provider gateways", + let evidence_error = write_lifecycle_evidence(context, &facts) + .err() + .map(|error| error.to_string()); + facts.insert("evidence_write_error".to_owned(), serde_json::json!(evidence_error)); + + if let Some(error) = body.as_ref().err() { + return proof_failure( + &format!("provider withdrawal lifecycle failed: {error}"), facts, start.elapsed(), - )) - } else { - Ok(proof_failure( - &format!( - "provider gateway distribution or overlay stability failed: counts={counts:?}, overlay_changes={overlay_changes:?}" - ), + ); + } + if let Some(error) = restoration_error { + return proof_failure( + &format!("provider selectors did not restore: {error}"), facts, start.elapsed(), - )) + ); + } + if let Some(error) = evidence_error { + return proof_failure( + &format!("could not persist lifecycle evidence: {error}"), + facts, + start.elapsed(), + ); } + proof_success( + "partial fallback, empty overlay serving, no backend contact, bound-session denial, and provider restoration passed", + facts, + start.elapsed(), + ) } /// Run one proof assertion and retain failures as structured evidence. @@ -2512,7 +4765,7 @@ fn configure_swim_peers(_forge_bin: &Path, _resolved_config: &Path) -> Result<() } for cluster in CLUSTERS { - let ctx = format!("kind-grid-provider-traffic-{cluster}"); + let ctx = cluster_context(cluster); wait_for_deployment("grid-operator", GRID_SYSTEM_NS, &ctx)?; eprintln!(" [OK] {cluster}: operator restarted with SWIM config"); } @@ -2522,7 +4775,7 @@ fn configure_swim_peers(_forge_bin: &Path, _resolved_config: &Path) -> Result<() /// Read the SWIM `LoadBalancer` IP for a cluster directly from Kubernetes. fn read_swim_lb_ip(cluster: &str) -> Result> { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let output = Command::new("kubectl") .args([ "--context", @@ -2563,7 +4816,7 @@ fn update_operator_swim_config( advertise_ip: &str, seeds: &str, ) -> Result<(), Box> { - let context = format!("kind-grid-provider-traffic-{cluster}"); + let context = cluster_context(cluster); let operator_image = std::env::var("GRID_XTASK_OPERATOR_IMAGE") .unwrap_or_else(|_| "grid-operator:provider-traffic-qualification".to_owned()); @@ -2664,15 +4917,16 @@ fn teardown_environment(context: &ProviderTrafficContext) -> Result<(), Box Result<(), Box Result<(), Box> { let mode = options.mode(); - let run_id = format_utc_timestamp(); + let run_id = format!("{}-{}", format_utc_timestamp(), std::process::id()); + let cluster_prefix = format!("grid258-{run_id}"); + RUN_CLUSTER_PREFIX + .set(cluster_prefix.clone()) + .map_err(|error| format!("provider-traffic cluster prefix was already initialized: {error:?}"))?; let wall_start = Instant::now(); let _started_at = format_utc_iso(); let evidence_dir = resolve_evidence_dir(forge_config, options, &run_id)?; fs::create_dir_all(&evidence_dir)?; - let setup_ctx = prepare_setup(forge_config); + let setup_ctx = prepare_setup(forge_config, &evidence_dir, &run_id, &cluster_prefix); let mut teardown_success = false; let mut run_error = None; let mut overlay_state = OverlayState::default(); @@ -2708,19 +4966,46 @@ pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), B match deploy_setup(context) { Ok(state) => { overlay_state = state; - image_evidence = match collect_image_evidence() { - Ok(collected_images) => collected_images, - Err(error) => { - run_error = Some(format!("image evidence collection failed: {error}")); - BTreeMap::new() - }, - }; eprintln!(); eprintln!("{OUTPUT_RULE}"); eprintln!("ENVIRONMENT READY - Starting proof scenarios"); eprintln!("{OUTPUT_RULE}"); - let scenario_results = run_provider_traffic_scenarios(); + let generated_config_result = (mode == DemoMode::Full + && provider_traffic_scenario_names(mode).contains(&"generated_consumer_config_convergence")) + .then(|| { + let assertion = run_assertion("generated_consumer_config_convergence", || { + let started = Instant::now(); + match activate_generated_consumer_config() { + Ok(evidence) => Ok(proof_success( + "live consumer loaded the generated overlay and projected credential filter, then routed a credential-bearing request without a client provider secret", + BTreeMap::from([("runtime_evidence".to_owned(), evidence)]), + started.elapsed(), + )), + Err(error) => Ok(proof_failure( + &format!("operator-generated consumer config did not converge: {error}"), + BTreeMap::from([( + "configuration_error".to_owned(), + serde_json::json!(error.to_string()), + )]), + started.elapsed(), + )), + } + }); + match assertion { + Ok(result) => result, + Err(error) => proof_failure( + &format!("generated consumer config assertion errored: {error}"), + BTreeMap::new(), + Duration::ZERO, + ), + } + }); + + let mut scenario_results = run_provider_traffic_scenarios(mode, context); + if let Some(result) = generated_config_result { + scenario_results.insert("generated_consumer_config_convergence".to_owned(), result); + } let failed_proofs: Vec<&str> = scenario_results .iter() @@ -2730,6 +5015,18 @@ pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), B run_error = Some(format!("runtime proofs failed: {}", failed_proofs.join(", "))); } + image_evidence = match collect_image_evidence() { + Ok(collected_images) => collected_images, + Err(error) => { + let message = format!("image evidence collection failed: {error}"); + run_error = Some(match run_error.take() { + Some(previous) => format!("{previous}; {message}"), + None => message, + }); + BTreeMap::new() + }, + }; + // Teardown if requested if options.teardown && (run_error.is_none() || !options.keep_on_failure) { match teardown_environment(context) { @@ -2804,100 +5101,230 @@ pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), B } /// Collect actual image evidence from the deployed clusters. -#[expect( - clippy::too_many_lines, - reason = "Evidence collection queries the bounded set of deployed component images." -)] fn collect_image_evidence() -> Result, Box> { let mut image_evidence = BTreeMap::new(); for cluster in CLUSTERS { - let context = format!("kind-grid-provider-traffic-{cluster}"); - - // Get grid operator image - let operator_output = Command::new("kubectl") - .args([ - "get", - "deployment/grid-operator", - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; - - if operator_output.status.success() { - let operator_image = String::from_utf8_lossy(&operator_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_operator"), operator_image); + let context = cluster_context(cluster); + for (key, deployment) in image_evidence_deployments(cluster) { + image_evidence.insert( + format!("{cluster}_{key}"), + deployment_runtime_image_evidence(&context, &deployment)?, + ); } + } - // Get consumer gateway image - let consumer_output = Command::new("kubectl") - .args([ - "get", - "deployment/consumer-gateway", - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; + if let Some(embedded_image) = collect_embedded_gateway_image_evidence()? { + image_evidence.insert("provider-a_embedded_grid_gateway".to_owned(), embedded_image); + } - if consumer_output.status.success() { - let consumer_image = String::from_utf8_lossy(&consumer_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_consumer_gateway"), consumer_image); - } + Ok(image_evidence) +} - // Get provider gateway image - let provider_output = Command::new("kubectl") - .args([ - "get", - "deployment/provider-gateway", - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; +/// Capture the optional embedded Grid gateway, which is installed only on the consumer site. +fn collect_embedded_gateway_image_evidence() -> Result, Box> { + let consumer_context = cluster_context(CONSUMER_SITE); + if deployment_exists(&consumer_context, GRID_SERVING_GATEWAY)? { + return Ok(Some(deployment_runtime_image_evidence( + &consumer_context, + GRID_SERVING_GATEWAY, + )?)); + } - if provider_output.status.success() { - let provider_image = String::from_utf8_lossy(&provider_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_provider_gateway"), provider_image); - } + Ok(None) +} - // Get VCR inference image - let vcr_output = Command::new("kubectl") - .args([ - "get", - &format!("deployment/vcr-inference-{cluster}"), - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; +/// Check whether a deployment exists without treating a missing optional workload as an error. +fn deployment_exists(context: &str, deployment: &str) -> Result> { + let output = Command::new("kubectl") + .args([ + "get", + &format!("deployment/{deployment}"), + "--context", + context, + "-n", + GRID_SYSTEM_NS, + "--ignore-not-found", + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err(format!( + "could not check for embedded gateway image evidence: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + .into()); + } + let deployment_json: serde_json::Value = serde_json::from_slice(&output.stdout)?; + Ok(deployment_json + .pointer("/metadata/name") + .and_then(serde_json::Value::as_str) + .is_some()) +} - if vcr_output.status.success() { - let mock_image = String::from_utf8_lossy(&vcr_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_vcr_inference"), mock_image); - } +/// Return deployments that belong to one provider-traffic cluster. +fn image_evidence_deployments(cluster: &str) -> Vec<(&'static str, String)> { + let mut deployments = vec![ + ("operator", "grid-operator".to_owned()), + ("provider_gateway", "provider-gateway".to_owned()), + ("simulator", format!("vcr-inference-{cluster}")), + ]; + if cluster == CONSUMER_SITE { + deployments.push(("consumer_gateway", "consumer-gateway".to_owned())); } + deployments +} - Ok(image_evidence) +/// Capture requested image references and Kubernetes-reported image IDs for every Ready pod of a deployment. +#[expect( + clippy::too_many_lines, + reason = "deployment identity, requested images, and every ready runtime image ID are captured together" +)] +fn deployment_runtime_image_evidence(context: &str, deployment: &str) -> Result> { + let deployment_output = Command::new("kubectl") + .args([ + "get", + &format!("deployment/{deployment}"), + "--context", + context, + "-n", + "grid-system", + "-o", + "json", + ]) + .output()?; + if !deployment_output.status.success() { + return Err(format!( + "could not read deployment {deployment}: {}", + String::from_utf8_lossy(&deployment_output.stderr).trim() + ) + .into()); + } + let deployment_json: serde_json::Value = serde_json::from_slice(&deployment_output.stdout)?; + let selector = deployment_json + .pointer("/spec/selector/matchLabels") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| format!("deployment {deployment} has no label selector"))?; + let requested = deployment_json + .pointer("/spec/template/spec/containers") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| format!("deployment {deployment} has no container template"))?; + let requested_by_name: BTreeMap<_, _> = requested + .iter() + .filter_map(|container| Some((container.get("name")?.as_str()?, container.get("image")?.as_str()?))) + .collect(); + let selector_arg = selector + .iter() + .filter_map(|(key, value)| Some(format!("{key}={}", value.as_str()?))) + .collect::>() + .join(","); + if selector_arg.is_empty() { + return Err(format!("deployment {deployment} has an empty label selector").into()); + } + let pod_output = Command::new("kubectl") + .args([ + "get", + "pods", + "-l", + &selector_arg, + "--context", + context, + "-n", + "grid-system", + "-o", + "json", + ]) + .output()?; + if !pod_output.status.success() { + return Err(format!( + "could not read pods for deployment {deployment}: {}", + String::from_utf8_lossy(&pod_output.stderr).trim() + ) + .into()); + } + let pods_json: serde_json::Value = serde_json::from_slice(&pod_output.stdout)?; + let pods = pods_json + .get("items") + .and_then(serde_json::Value::as_array) + .ok_or("kubectl pods response has no items array")?; + let mut ready_pods = Vec::new(); + for pod in pods { + let ready = pod + .pointer("/status/conditions") + .and_then(serde_json::Value::as_array) + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition.get("type").and_then(serde_json::Value::as_str) == Some("Ready") + && condition.get("status").and_then(serde_json::Value::as_str) == Some("True") + }) + }); + if !ready { + continue; + } + let pod_name = pod + .pointer("/metadata/name") + .and_then(serde_json::Value::as_str) + .ok_or("matching pod has no name")?; + let statuses = pod + .pointer("/status/containerStatuses") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| format!("Ready pod {pod_name} has no containerStatuses"))?; + let containers: Vec<_> = statuses + .iter() + .map(|status| { + let name = status + .get("name") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| format!("pod {pod_name} has unnamed container status"))?; + let image_id = status + .get("imageID") + .and_then(serde_json::Value::as_str) + .filter(|image_id| !image_id.is_empty()) + .ok_or_else(|| format!("pod {pod_name} container {name} has no runtime imageID"))?; + Ok(serde_json::json!({ + "name": name, + "requested": requested_by_name.get(name), + "imageID": image_id, + })) + }) + .collect::>()?; + ready_pods.push(serde_json::json!({ "pod": pod_name, "containers": containers })); + } + if ready_pods.is_empty() { + return Err(format!("deployment {deployment} has no Ready pod for runtime image evidence").into()); + } + ready_pods.sort_by(|left, right| left["pod"].as_str().cmp(&right["pod"].as_str())); + Ok(serde_json::to_string(&serde_json::json!({ "pods": ready_pods }))?) } #[cfg(test)] mod tests { use super::*; + #[test] + fn image_evidence_deployments_scope_consumer_to_consumer_site() { + let provider_a = image_evidence_deployments(CONSUMER_SITE); + assert!( + provider_a + .iter() + .any(|(key, deployment)| { *key == "consumer_gateway" && deployment == "consumer-gateway" }) + ); + + for site in ["provider-b", "provider-c"] { + let deployments = image_evidence_deployments(site); + assert!(!deployments.iter().any(|(key, _)| *key == "consumer_gateway")); + } + } + + #[test] + fn embedded_consumer_charts_resolve_from_grid_repository_root() { + let root = grid_repository_root().unwrap_or_else(|_| std::process::abort()); + assert!(root.join("charts/grid-operator/Chart.yaml").is_file()); + assert!(root.join("charts/praxis-gateway/Chart.yaml").is_file()); + assert!(!root.join("tests/e2e/topologies/grid-provider-traffic/charts").exists()); + } + #[test] fn proof_success_creation() { let mut facts = BTreeMap::new(); @@ -3019,6 +5446,86 @@ mod tests { assert_eq!(CLUSTERS, &["provider-a", "provider-b", "provider-c"]); } + #[test] + fn curl_status_parser_uses_explicit_marker_and_rejects_no_response() { + assert_eq!( + curl_http_status(b"HTTP/1.1 200 OK\r\n\r\nbody\nGRID258_HTTP_STATUS:200\n"), + Ok(200) + ); + assert_eq!(curl_http_status(b"GRID258_HTTP_STATUS:404\n"), Ok(404)); + assert_eq!( + curl_http_status(b"GRID258_HTTP_STATUS:000\n"), + Err("curl received no HTTP response (status 000)".to_owned()) + ); + assert_eq!( + curl_http_status(b"curl: (7) connection refused"), + Err("curl output did not contain its HTTP status marker".to_owned()) + ); + } + + #[test] + fn serving_revision_matcher_handles_colored_tracing_fields() { + let logs = "\x1b[2mINFO\x1b[0m grid_serving_revision_serving revision=abc123 \x1b[3m candidate_count\x1b[0m\x1b[3m=\x1b[0m3"; + let line = embedded_serving_revision_lines(logs, "abc123", 3) + .into_iter() + .next() + .unwrap_or_default(); + assert!(line.contains("candidate_count=3")); + assert!(embedded_serving_revision_lines(logs, "wrong", 3).is_empty()); + assert!(embedded_serving_revision_lines(logs, "abc123", 2).is_empty()); + } + + #[test] + fn restored_revision_wait_requires_a_new_acceptance_after_its_old_log() { + let old = "INFO grid_serving_revision_serving revision=abc123 candidate_count=3"; + assert!(!has_new_embedded_serving_revision_observation(old, "abc123", 3, 1)); + let updated = format!("{old}\nINFO grid_serving_revision_serving revision=abc123 candidate_count=3"); + assert!(has_new_embedded_serving_revision_observation(&updated, "abc123", 3, 1)); + assert!(!has_new_embedded_serving_revision_observation(&updated, "abc123", 2, 1)); + } + + #[test] + fn full_mode_appends_lifecycle_without_changing_the_six_existing_scenarios() { + let quick = provider_traffic_scenario_names(DemoMode::Quick); + let full = provider_traffic_scenario_names(DemoMode::Full); + assert_eq!(quick.len(), 6); + assert_eq!(full.len(), 9); + assert_eq!(full.get(..6), Some(quick.as_slice())); + assert_eq!(full.get(6), Some(&"generated_consumer_config_convergence")); + assert_eq!(full.get(7), Some(&"embedded_serving_gateway_convergence")); + assert_eq!(full.last(), Some(&"provider_withdrawal_lifecycle")); + } + + #[test] + fn resolved_paths_and_kind_contexts_are_run_scoped() { + let mut config: serde_yaml::Value = serde_yaml::from_str( + "spec:\n runtime:\n clusterPrefix: old\n stacks:\n test:\n steps:\n - type: template-file\n target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml\n source: configs/consumer/praxis.yaml\n - type: exec\n command: kubectl --context kind-grid-provider-traffic-{{ cluster.name }} --from-file=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml\n", + ) + .unwrap_or_else(|_| std::process::abort()); + rewrite_run_scoped_paths(&mut config, "grid258-unique-run", Path::new("/tmp/run/forge-state")); + let resolved = serde_yaml::to_string(&config).unwrap_or_else(|_| std::process::abort()); + assert!(resolved.contains("target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml")); + assert!(resolved.contains("/tmp/run/forge-state/runtime/{{ cluster.name }}/consumer/praxis.yaml")); + assert!(resolved.contains("kind-grid258-unique-run-{{ cluster.name }}")); + assert!(!resolved.contains("kind-grid-provider-traffic-")); + } + + #[test] + fn forge_environment_and_network_name_are_run_scoped() { + let mut config: serde_yaml::Value = serde_yaml::from_str( + "metadata:\n name: grid-provider-traffic\nspec:\n runtime:\n clusterPrefix: old\n", + ) + .unwrap_or_else(|_| std::process::abort()); + scope_environment_name(&mut config, "run-1234").unwrap_or_else(|_| std::process::abort()); + let name = config + .get("metadata") + .and_then(|metadata| metadata.get("name")) + .and_then(serde_yaml::Value::as_str) + .unwrap_or_else(|| std::process::abort()); + assert_eq!(name, "grid-provider-traffic-run-1234"); + assert_eq!(format!("{name}-net"), "grid-provider-traffic-run-1234-net"); + } + #[test] fn evidence_schema_version() { assert_eq!(EVIDENCE_SCHEMA_VERSION, "1"); @@ -3054,6 +5561,70 @@ mod tests { ); } + #[test] + fn embedded_gateway_reference_declares_mtls_provider_hops() { + let endpoints = PROVIDER_RESOURCES + .iter() + .map(|(site, resource)| { + serde_json::json!({ + "cluster": resource, + "address": format!("{site}.grid.internal:8443"), + "transport": { "mode": "mutual_tls", "sni": format!("{site}.grid.internal") } + }) + }) + .collect::>(); + let gateway = embedded_serving_gateway_ref(&endpoints); + assert_eq!( + gateway + .pointer("/consumerConfig/enabled") + .and_then(serde_json::Value::as_bool), + Some(false) + ); + let rendered_endpoints = gateway + .pointer("/consumerConfig/clusterEndpoints") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| std::process::abort()); + assert_eq!(rendered_endpoints.len(), PROVIDER_RESOURCES.len()); + assert!(rendered_endpoints.iter().all(|endpoint| { + endpoint.pointer("/transport/mode").and_then(serde_json::Value::as_str) == Some("mutual_tls") + && endpoint + .pointer("/transport/sni") + .and_then(serde_json::Value::as_str) + .is_some() + })); + } + + #[test] + fn embedded_serving_snapshot_requires_the_exact_provider_hop_allowlist() { + let expected: Vec<_> = PROVIDER_RESOURCES.iter().map(|(_, resource)| *resource).collect(); + let valid = serde_json::json!({ "serving_config": { "provider_hop_clusters": expected } }); + assert!(matches!(require_serving_provider_hops(&valid), Ok(()))); + + let missing = serde_json::json!({ "serving_config": { "provider_hop_clusters": [] } }); + assert!(require_serving_provider_hops(&missing).is_err()); + let extra = serde_json::json!({ + "serving_config": { + "provider_hop_clusters": ["vcr-provider-a-provider", "vcr-provider-b-provider", "vcr-provider-c-provider", "untrusted"] + } + }); + assert!(require_serving_provider_hops(&extra).is_err()); + } + + #[test] + fn restored_provider_attribution_uses_site_labels_not_cluster_names() { + let sites = expected_provider_attributions(); + assert_eq!( + sites, + BTreeSet::from([ + "provider-a".to_owned(), + "provider-b".to_owned(), + "provider-c".to_owned() + ]) + ); + assert!(sites.contains("provider-a")); + assert!(!sites.contains("vcr-provider-a-provider")); + } + #[test] fn provider_traffic_constants_describe_focused_topology() { assert_eq!(CLUSTERS, &["provider-a", "provider-b", "provider-c"]); diff --git a/xtask/src/env/static_weighted_qualification.rs b/xtask/src/env/static_weighted_qualification.rs index 4b759b4b3..7a68facf9 100644 --- a/xtask/src/env/static_weighted_qualification.rs +++ b/xtask/src/env/static_weighted_qualification.rs @@ -19,9 +19,6 @@ use super::{DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, safe_trunca // Constants // ----------------------------------------------------------------------------- -/// Directory where generated TLS certificates are stored. -const CERTS_DIR: &str = "tests/env/certs"; - /// Ordered provider-site cluster names in the static-weighted scenario. /// /// The consumer entrypoint is deployed only in `CONSUMER_SITE`; the other @@ -68,6 +65,8 @@ const SETUP_PHASES: usize = 14; static PROBE_SEQUENCE: AtomicU64 = AtomicU64::new(0); /// Run-scoped Forge prefix, initialized once by the qualification entrypoint. static RUN_NAME: OnceLock = OnceLock::new(); +/// Keep generated Kind control-plane node names below Kubernetes' 63-byte label-value limit. +const RUN_NAME_PREFIX: &str = "grid258"; /// Persistent client pod used for all weighted traffic samples. const STATIC_CLIENT_POD: &str = "static-weighted-client"; @@ -76,14 +75,68 @@ fn run_name() -> &'static str { RUN_NAME.get().map_or(BASE_RUN_NAME, |value| value.as_str()) } +/// Use a compact physical prefix: Kind appends `-provider-a-control-plane` to the cluster name. +fn scoped_run_name(run_id: &str) -> String { + format!("{RUN_NAME_PREFIX}-{run_id}") +} + +/// Return the longest node name Kind will generate for this run's cluster names. +fn longest_kind_node_name_len(run_name: &str) -> usize { + CLUSTERS + .iter() + .map(|cluster| format!("{run_name}-{cluster}-control-plane").len()) + .max() + .unwrap_or_default() +} + /// Build the kubectl context for one run-scoped provider cluster. fn cluster_context(cluster: &str) -> String { format!("kind-{}-{cluster}", run_name()) } -/// Return the run-scoped consumer overlay `ConfigMap` name. -fn overlay_configmap() -> String { - format!("grid-overlay-{}-consumer-gateway", run_name()) +/// Resolve the consumer overlay `ConfigMap` named by the `GridNetwork`'s observed status. +fn overlay_configmap(cluster: &str) -> Result> { + let context = cluster_context(cluster); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + run_name(), + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err(format!("{cluster}: GridNetwork status is unavailable for overlay discovery").into()); + } + let grid_network: serde_json::Value = serde_json::from_slice(&output.stdout)?; + overlay_configmap_from_status(&grid_network) +} + +/// Select the `ConfigMap` for the consumer gateway from the `GridNetwork` status entries. +fn overlay_configmap_from_status(grid_network: &serde_json::Value) -> Result> { + let overlays = grid_network + .pointer("/status/overlayStatus") + .and_then(serde_json::Value::as_array) + .ok_or("GridNetwork status has no overlayStatus entries")?; + overlays + .iter() + .find(|overlay| { + overlay.pointer("/gatewayName").and_then(serde_json::Value::as_str) == Some("consumer-gateway") + && overlay.pointer("/namespace").and_then(serde_json::Value::as_str) == Some(GRID_SYSTEM_NS) + }) + .and_then(|overlay| overlay.pointer("/configMapName").and_then(serde_json::Value::as_str)) + .map(str::to_owned) + .ok_or_else(|| "GridNetwork status has no distributed consumer-gateway ConfigMap name".into()) +} + +/// Resolve the evidence directory so Forge runtime paths work from its topology working directory. +fn canonicalize_evidence_dir(evidence_dir: &Path) -> std::io::Result { + fs::canonicalize(evidence_dir) } // ----------------------------------------------------------------------------- @@ -97,10 +150,24 @@ struct ProviderTrafficContext { demo_root: PathBuf, /// Path to the resolved Forge config. resolved_config: PathBuf, + /// Isolated Forge state and generated runtime directory. + forge_state_dir: PathBuf, + /// TLS files used only by this run. + certs_dir: PathBuf, /// Path to the forge binary. forge_bin: PathBuf, } +/// Build a Forge command pinned to this run's state directory. +fn forge_command(context: &ProviderTrafficContext) -> Command { + let mut command = Command::new(&context.forge_bin); + command + .args(["--state-dir"]) + .arg(&context.forge_state_dir) + .env("FORGE_STATE_DIR", &context.forge_state_dir); + command +} + // ----------------------------------------------------------------------------- // Overlay State // ----------------------------------------------------------------------------- @@ -1154,13 +1221,14 @@ fn assert_overlay_acceptance() -> AssertionResult { { let cluster = CONSUMER_SITE; let context = cluster_context(cluster); + let overlay_name = overlay_configmap(cluster)?; // Step 1: Overlay ConfigMap exists let overlay_output = Command::new("kubectl") .args([ "get", "configmap", - &overlay_configmap(), + &overlay_name, "--context", &context, "-n", @@ -1178,7 +1246,7 @@ fn assert_overlay_acceptance() -> AssertionResult { .args([ "get", "configmap", - &overlay_configmap(), + &overlay_name, "--context", &context, "-n", @@ -1313,68 +1381,33 @@ fn load_images_into_clusters() -> Result<(), Box> { let operator = std::env::var("GRID_XTASK_OPERATOR_IMAGE") .unwrap_or_else(|_| "grid-operator:static-weighted-qualification".to_owned()); let vcr = crate::env::image_overrides::sim_image(); + let overlay_sync = crate::env::image_overrides::overlay_sync_image(); - for image in [&gateway, &operator, &vcr] { + for image in [&gateway, &operator, &vcr, &overlay_sync] { require_local_image(image)?; eprintln!(" verified local image: {image}"); } for cluster in CLUSTERS { - for image in [&gateway, &operator, &vcr] { + for image in [&gateway, &operator, &vcr, &overlay_sync] { eprintln!(" loading {image} into {cluster}..."); - load_docker_image_into_kind(image, &format!("{}-{cluster}", run_name()))?; + crate::env::image_overrides::load_docker_image_into_kind(image, &format!("{}-{cluster}", run_name()))?; } eprintln!(" [OK] {cluster}: all images loaded"); } Ok(()) } -/// Import one host image into a run-owned Kind node using only its host -/// platform. This avoids OCI-index imports failing when the local Docker -/// store has only the linux/amd64 child content. -fn load_docker_image_into_kind(image: &str, kind_name: &str) -> Result<(), Box> { - let control_plane = format!("{kind_name}-control-plane"); - let mut save = Command::new("docker") - .args(["save", "--platform", "linux/amd64", image]) - .stdout(Stdio::piped()) - .spawn()?; - let save_stdout = save.stdout.take().ok_or("docker save did not provide stdout")?; - let import_status = Command::new("docker") - .args([ - "exec", - "--privileged", - "-i", - &control_plane, - "ctr", - "--namespace=k8s.io", - "images", - "import", - "--digests", - "--snapshotter=overlayfs", - "-", - ]) - .stdin(save_stdout) - .status()?; - let save_status = save.wait()?; - if !save_status.success() { - return Err(format!("docker save failed for {image}").into()); - } - if !import_status.success() { - return Err(format!("failed to import {image} into {control_plane}").into()); - } - Ok(()) -} - /// Generate TLS certificates for all static-weighted identities. /// /// Must be called BEFORE `forge up` so the certificates exist on the host /// when `install_provider_boundary` creates the Kubernetes Secrets. -fn stage_provider_boundary() -> Result<(), Box> { +fn stage_provider_boundary(certs_dir: &Path) -> Result<(), Box> { let identities: Vec = CLUSTERS.iter().map(|c| (*c).to_owned()).collect(); - certs::generate_all(&identities)?; + certs::generate_all_in_dir(&identities, certs_dir)?; let wrong_ca = ::certs::generate_ca("Combined Site untrusted test CA")?; - fs::write(Path::new(CERTS_DIR).join("untrusted-ca.pem"), wrong_ca.cert_pem)?; + fs::write(certs_dir.join("untrusted-ca.pem"), wrong_ca.cert_pem)?; eprintln!(" [OK] TLS certificates generated for provider-a, provider-b, provider-c"); Ok(()) @@ -1384,9 +1417,7 @@ fn stage_provider_boundary() -> Result<(), Box> { /// /// Must be called AFTER `forge up` since the clusters must exist. Gateway /// deployments are restarted so pods pick up the new volume mounts. -fn install_provider_boundary() -> Result<(), Box> { - let certs_dir = Path::new(CERTS_DIR); - +fn install_provider_boundary(certs_dir: &Path) -> Result<(), Box> { for cluster in CLUSTERS { let context = cluster_context(cluster); @@ -1478,6 +1509,7 @@ fn apply_credential_secret(context: &str, secret_name: &str, token: &str) -> Res )] fn read_cluster_overlay(cluster: &str) -> Result> { let context = cluster_context(cluster); + let overlay_name = overlay_configmap(cluster)?; let output = Command::new("kubectl") .args([ @@ -1487,7 +1519,7 @@ fn read_cluster_overlay(cluster: &str) -> Result Result, Box> { let overlay = read_cluster_overlay("provider-a")?; + let overlay_name = overlay_configmap("provider-a")?; let configmap = Command::new("kubectl") .args([ "--context", @@ -1613,7 +1646,7 @@ fn wait_for_static_readiness( GRID_SYSTEM_NS, "get", "configmap", - &overlay_configmap(), + &overlay_name, "-o", "json", ]) @@ -2071,21 +2104,24 @@ fn collect_overlay_diagnostics() { ); eprintln!(" [DIAG] {cluster}: 6. Overlay ConfigMap content"); - drop( - Command::new("kubectl") - .args([ - "--context", - &context, - "-n", - GRID_SYSTEM_NS, - "get", - "configmap", - &overlay_configmap(), - "-o", - "json", - ]) - .status(), - ); + match overlay_configmap(cluster) { + Ok(overlay_name) => drop( + Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "configmap", + &overlay_name, + "-o", + "json", + ]) + .status(), + ), + Err(error) => eprintln!(" [DIAG] {cluster}: could not resolve overlay ConfigMap: {error}"), + } eprintln!(" [DIAG] {cluster}: 7. InferenceProvider CRs"); drop( @@ -2250,18 +2286,79 @@ fn materialize_provider_config( } /// Materialize the Forge configuration with image overrides. -fn materialize_config(source: &Path) -> Result> { +fn materialize_config( + source: &Path, + run_id: &str, + forge_state_dir: &Path, + evidence_dir: &Path, +) -> Result> { let content = fs::read_to_string(source)?; let mut config: serde_yaml::Value = serde_yaml::from_str(&content)?; rewrite_run_scope(&mut config); - apply_image_overrides(&mut config); + rewrite_forge_exec_runtime_paths(&mut config, forge_state_dir); + apply_image_overrides(&mut config)?; let rendered = serde_yaml::to_string(&config)?; let parent = source.parent().ok_or("source config must have parent directory")?; - let output = parent.join(".forge.resolved.yaml"); + let output = parent.join(format!(".forge.resolved-{run_id}.yaml")); fs::write(&output, rendered)?; + fs::copy(&output, evidence_dir.join("resolved-forge.yaml"))?; Ok(output) } +/// Rewrite runtime paths only in `exec` commands; Forge keeps template targets relative to its state directory. +fn rewrite_forge_exec_runtime_paths(value: &mut serde_yaml::Value, forge_state_dir: &Path) { + match value { + serde_yaml::Value::Mapping(mapping) => { + let is_exec = mapping + .get(serde_yaml::Value::String("type".to_owned())) + .and_then(serde_yaml::Value::as_str) + == Some("exec"); + if is_exec { + if let Some(command) = mapping.get_mut(serde_yaml::Value::String("command".to_owned())) { + rewrite_forge_exec_path_strings(command, forge_state_dir); + } + } else { + for child in mapping.values_mut() { + rewrite_forge_exec_runtime_paths(child, forge_state_dir); + } + } + }, + serde_yaml::Value::Sequence(items) => { + for item in items { + rewrite_forge_exec_runtime_paths(item, forge_state_dir); + } + }, + serde_yaml::Value::Null + | serde_yaml::Value::Bool(_) + | serde_yaml::Value::Number(_) + | serde_yaml::Value::String(_) + | serde_yaml::Value::Tagged(_) => {}, + } +} + +/// Replace Forge-relative runtime references inside a shell command with this run's absolute state path. +fn rewrite_forge_exec_path_strings(value: &mut serde_yaml::Value, forge_state_dir: &Path) { + match value { + serde_yaml::Value::String(command) => { + *command = command.replace(".forge/runtime/", &format!("{}/runtime/", forge_state_dir.display())); + }, + serde_yaml::Value::Sequence(items) => { + for item in items { + rewrite_forge_exec_path_strings(item, forge_state_dir); + } + }, + serde_yaml::Value::Mapping(mapping) => { + for child in mapping.values_mut() { + rewrite_forge_exec_path_strings(child, forge_state_dir); + } + }, + serde_yaml::Value::Null + | serde_yaml::Value::Bool(_) + | serde_yaml::Value::Number(_) + | serde_yaml::Value::Tagged(_) => {}, + } +} + /// Rewrite topology-owned names to the unique prefix for this qualification run. fn rewrite_run_scope(value: &mut serde_yaml::Value) { match value { @@ -2281,51 +2378,18 @@ fn rewrite_run_scope(value: &mut serde_yaml::Value) { } } -/// Apply image overrides from environment variables to the Forge configuration. -#[expect( - clippy::too_many_lines, - clippy::collapsible_if, - reason = "Image override application with structured YAML manipulation; nested ifs follow YAML structure hierarchy" -)] -fn apply_image_overrides(config: &mut serde_yaml::Value) { - let gateway_image = std::env::var("GRID_XTASK_GATEWAY_IMAGE") - .unwrap_or_else(|_| "praxis-ai:static-weighted-qualification".to_owned()); - let operator_image = std::env::var("GRID_XTASK_OPERATOR_IMAGE") - .unwrap_or_else(|_| "grid-operator:static-weighted-qualification".to_owned()); - let vcr_image = crate::env::image_overrides::sim_image(); - let image_pull_policy = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()); - - let (gateway_repo, gateway_tag) = parse_image_ref(&gateway_image); - let (operator_repo, operator_tag) = parse_image_ref(&operator_image); - - if let Some(spec) = config.get_mut("spec") { - if let Some(clusters) = spec.get_mut("clusters") { - if let Some(clusters_array) = clusters.as_sequence_mut() { - for cluster in clusters_array { - if let Some(properties) = cluster.get_mut("properties") { - if let Some(props_map) = properties.as_mapping_mut() { - let pairs = [ - ("gatewayImage", &gateway_image), - ("operatorImage", &operator_image), - ("vcrImage", &vcr_image), - ("imagePullPolicy", &image_pull_policy), - ("gatewayImageRepo", &gateway_repo), - ("gatewayImageTag", &gateway_tag), - ("operatorImageRepo", &operator_repo), - ("operatorImageTag", &operator_tag), - ]; - for (key, val) in pairs { - props_map.insert( - serde_yaml::Value::String(key.to_owned()), - serde_yaml::Value::String(val.clone()), - ); - } - } - } - } - } - } - } +/// Apply shared image overrides, including the overlay-sync image. +fn apply_image_overrides(config: &mut serde_yaml::Value) -> Result<(), Box> { + let images = crate::env::forge_config::ImageOverrides { + gateway: std::env::var("GRID_XTASK_GATEWAY_IMAGE") + .unwrap_or_else(|_| "praxis-ai:static-weighted-qualification".to_owned()), + operator: std::env::var("GRID_XTASK_OPERATOR_IMAGE") + .unwrap_or_else(|_| "grid-operator:static-weighted-qualification".to_owned()), + overlay_sync: crate::env::image_overrides::overlay_sync_image(), + vcr: crate::env::image_overrides::sim_image(), + pull_policy: std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()), + }; + crate::env::forge_config::apply_image_values(config, &images) } /// Parse image reference into (repo, tag) components. @@ -2341,21 +2405,20 @@ fn parse_image_ref(image: &str) -> (String, String) { } /// Prepare setup context from configuration. -fn prepare_setup(forge_config: &Path) -> Result> { +fn prepare_setup( + forge_config: &Path, + run_id: &str, + evidence_dir: &Path, +) -> Result> { let root = super::demo_root(forge_config); eprintln!("Forge config: {}", forge_config.display()); eprintln!("Demo root: {}", root.display()); - for generated in [Path::new(".forge/state.json"), Path::new(".forge/lock")] { - if generated.exists() { - fs::remove_file(generated).map_err(|error| format!("failed to clear Forge state: {error}"))?; - } - } - let generated_runtime = Path::new(".forge/runtime"); - if generated_runtime.exists() { - fs::remove_dir_all(generated_runtime) - .map_err(|error| format!("failed to clear generated Forge runtime: {error}"))?; - } - let resolved_config = materialize_config(forge_config)?; + let evidence_dir = canonicalize_evidence_dir(evidence_dir)?; + let forge_state_dir = evidence_dir.join("forge-state"); + let certs_dir = forge_state_dir.join("certs"); + fs::create_dir_all(&forge_state_dir)?; + fs::create_dir_all(&certs_dir)?; + let resolved_config = materialize_config(forge_config, run_id, &forge_state_dir, &evidence_dir)?; let forge_bin = glb::resolve_forge_binary() .ok_or("praxis-forge binary not found")? .into(); @@ -2363,6 +2426,8 @@ fn prepare_setup(forge_config: &Path) -> Result Result Result<(), Box> { +fn authorize_discovered_sites(certs_dir: &Path) -> Result<(), Box> { const TRUST_TIMEOUT: Duration = Duration::from_secs(120); let grid_network = run_name().to_owned(); @@ -2387,7 +2452,7 @@ fn authorize_discovered_sites() -> Result<(), Box> { } let site_name = format!("{grid_network}-{remote}"); operator::wait_for_auto_gridsite(&context, &site_name, &grid_network, TRUST_TIMEOUT)?; - let canonical_fp = certs::site_certificate_fingerprint(remote)?; + let canonical_fp = certs::site_certificate_fingerprint_in_dir(remote, certs_dir)?; operator::wait_for_expected_site_certificate(&context, &site_name, &canonical_fp, TRUST_TIMEOUT)?; let server_name = format!("{remote}.grid.internal"); operator::patch_gridsite_identity_trust(&context, &site_name, &canonical_fp, &server_name)?; @@ -2419,7 +2484,7 @@ fn deploy_setup(context: &ProviderTrafficContext) -> Result Result Result Result Result<(), Box> { + let apply_stack = |cluster: &str, stack: &str| -> Result<(), Box> { eprintln!(" applying {stack} to {cluster}..."); - let stack_status = Command::new(forge_bin) + let stack_status = forge_command(context) .arg("--config") - .arg(resolved_config) + .arg(&context.resolved_config) .args(["--non-interactive", "stack", "apply", cluster, stack]) .status()?; if !stack_status.success() { @@ -2491,11 +2552,11 @@ fn deploy_setup(context: &ProviderTrafficContext) -> Result Result Result Result Result Result Result Result<(), Box Result<(), Box Result<(), Box bool { + keep_on_failure && run_failed +} + /// Run the static-weighted qualification. #[expect( clippy::too_many_lines, @@ -3546,15 +3605,20 @@ fn write_evidence(path: &Path, evidence: &Evidence) -> Result<(), Box Result<(), Box> { let mode = options.mode(); - let run_id = format_utc_timestamp(); - drop(RUN_NAME.set(format!("{BASE_RUN_NAME}-{run_id}"))); + let run_id = format!("{}-{}", format_utc_timestamp(), std::process::id()); + let scoped_name = scoped_run_name(&run_id); + let longest_node_name = longest_kind_node_name_len(&scoped_name); + if longest_node_name > 63 { + return Err(format!("run-scoped Kind node name is {longest_node_name} bytes; maximum is 63").into()); + } + drop(RUN_NAME.set(scoped_name)); let wall_start = Instant::now(); let _started_at = format_utc_iso(); let evidence_dir = resolve_evidence_dir(forge_config, options, &run_id)?; fs::create_dir_all(&evidence_dir)?; - let setup_ctx = prepare_setup(forge_config); + let setup_ctx = prepare_setup(forge_config, &run_id, &evidence_dir); let mut run_error = None; let mut overlay_state = OverlayState::default(); let mut image_evidence = BTreeMap::new(); @@ -3638,7 +3702,7 @@ pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), B let evidence_file = evidence_dir.join("results.json"); write_evidence(&evidence_file, &evidence)?; - if options.teardown && !options.keep_on_failure { + if options.teardown && !should_preserve_after_failure(options.keep_on_failure, run_error.is_some()) { if let Ok(context) = &setup_ctx { match teardown_environment(context) { Ok(()) => { @@ -3769,6 +3833,78 @@ fn collect_image_evidence() -> Result, Box 63); + } + + #[test] + fn overlay_configmap_uses_operator_reported_hashed_name() { + let grid_network = serde_json::json!({ + "status": { + "overlayStatus": [ + { + "gatewayName": "other-gateway", + "namespace": GRID_SYSTEM_NS, + "configMapName": "unrelated" + }, + { + "gatewayName": "consumer-gateway", + "namespace": GRID_SYSTEM_NS, + "configMapName": "grid-overlay-grid-static-weighted-consumer-gateway-21ab964b" + } + ] + } + }); + + assert_eq!( + overlay_configmap_from_status(&grid_network).ok().as_deref(), + Some("grid-overlay-grid-static-weighted-consumer-gateway-21ab964b") + ); + } + + #[test] + fn overlay_configmap_requires_distributed_consumer_status() { + let grid_network = serde_json::json!({ "status": { "overlayStatus": [] } }); + assert_eq!( + overlay_configmap_from_status(&grid_network) + .err() + .map(|error| error.to_string()), + Some("GridNetwork status has no distributed consumer-gateway ConfigMap name".to_owned()) + ); + } + + #[test] + fn forge_runtime_targets_stay_relative_and_exec_sources_use_absolute_state_paths() { + let Ok(evidence_dir) = canonicalize_evidence_dir(Path::new(".")) else { + std::process::abort(); + }; + let forge_state_dir = evidence_dir.join("forge-state"); + let mut config: serde_yaml::Value = serde_yaml::from_str( + "stacks:\n consumer:\n steps:\n - type: template-file\n target: .forge/runtime/provider-a/consumer/praxis.yaml\n - type: exec\n command: kubectl --from-file=.forge/runtime/provider-a/consumer/praxis.yaml\n", + ) + .unwrap_or_else(|_| std::process::abort()); + rewrite_forge_exec_runtime_paths(&mut config, &forge_state_dir); + let expected = forge_state_dir + .join("runtime/provider-a/consumer/praxis.yaml") + .display() + .to_string(); + assert!(Path::new(&expected).is_absolute()); + let rendered = serde_yaml::to_string(&config).unwrap_or_else(|_| std::process::abort()); + assert!(rendered.contains("target: .forge/runtime/provider-a/consumer/praxis.yaml")); + assert!(rendered.contains(&expected)); + } + #[test] fn proof_success_creation() { let mut facts = BTreeMap::new(); From a182045e08bb582bab176f0b45f99b2bf57bdb72 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Fri, 2 Oct 2026 19:10:53 +0000 Subject: [PATCH 02/10] fix(routing): separate embedded provider-hop configuration Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 40 ++++++++++++++++ deploy/crds/gridnetwork.yaml | 40 ++++++++++++++++ docs/architecture/consumer-config.md | 20 +++++--- docs/architecture/crds.md | 16 ++++++- operator/src/controller/grid_network.rs | 46 ++++++++++++++++--- operator/src/crd/grid_network.rs | 27 +++++++++++ operator/src/resources/consumer_config.rs | 6 +-- .../src/env/provider_traffic_qualification.rs | 25 ++++++++-- 8 files changed, 199 insertions(+), 21 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index 29aa1e050..2ec400c38 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -449,6 +449,46 @@ spec: namespace: description: Gateway namespace. type: string + providerHopEndpoints: + description: |- + Explicit mTLS provider-hop endpoints for the embedded `grid-gateway` + serving config. This allowlist is independent of `consumerConfig`, so a + missing or disabled generated Praxis config cannot alter embedded + serving behavior. Every entry must declare `mutual_tls` and a nonblank + SNI; the embedded gateway's upstream configuration must use that + verified TLS identity. + items: + description: |- + Explicit verified-mTLS endpoint identity trusted for embedded provider-hop + context. This does not configure the gateway's upstream TLS connection; the + embedded gateway must separately use verified TLS with this SNI. + properties: + cluster: + description: Cluster name used by the embedded gateway's `load_balancer` filter. + type: string + transport: + description: TLS mode and SNI configured for that embedded gateway upstream. + properties: + mode: + description: 'Transport mode: `mutual_tls` or `plaintext`.' + enum: + - mutual_tls + - plaintext + type: string + sni: + description: |- + TLS Server Name Indication (required when mode is `mutual_tls`; + must not be set when mode is `plaintext`). + nullable: true + type: string + required: + - mode + type: object + required: + - cluster + - transport + type: object + type: array supportsEmptyOverlay: default: false description: |- diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index a52c365dc..20232b449 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -441,6 +441,46 @@ spec: namespace: description: Gateway namespace. type: string + providerHopEndpoints: + description: |- + Explicit mTLS provider-hop endpoints for the embedded `grid-gateway` + serving config. This allowlist is independent of `consumerConfig`, so a + missing or disabled generated Praxis config cannot alter embedded + serving behavior. Every entry must declare `mutual_tls` and a nonblank + SNI; the embedded gateway's upstream configuration must use that + verified TLS identity. + items: + description: |- + Explicit verified-mTLS endpoint identity trusted for embedded provider-hop + context. This does not configure the gateway's upstream TLS connection; the + embedded gateway must separately use verified TLS with this SNI. + properties: + cluster: + description: Cluster name used by the embedded gateway's `load_balancer` filter. + type: string + transport: + description: TLS mode and SNI configured for that embedded gateway upstream. + properties: + mode: + description: 'Transport mode: `mutual_tls` or `plaintext`.' + enum: + - mutual_tls + - plaintext + type: string + sni: + description: |- + TLS Server Name Indication (required when mode is `mutual_tls`; + must not be set when mode is `plaintext`). + nullable: true + type: string + required: + - mode + type: object + required: + - cluster + - transport + type: object + type: array supportsEmptyOverlay: default: false description: |- diff --git a/docs/architecture/consumer-config.md b/docs/architecture/consumer-config.md index 65387fefc..2511b82de 100644 --- a/docs/architecture/consumer-config.md +++ b/docs/architecture/consumer-config.md @@ -179,8 +179,12 @@ injection table still require the consumer owner to roll/reload Praxis after the generated `praxis.yaml` changes. That rollout is separate from route-only overlay reloads. If credentials are present, a restored provider whose credential reference changed requires this config rollout before its request -can succeed; `credential_inject` fails closed when the configured reference -does not match the overlay. +can succeed when using a static credential table; `credential_inject` fails +closed when the configured reference does not match the overlay. In +projected-credential mode, a changed reference needs no config rollout when +the credential filter is already running and the matching Secret is mounted +under `{credentialMountBase}/{namespace}/{name}`. If that Secret projection +is not mounted, the request fails closed until the consumer installs it. See [`docs/architecture/crds.md`](crds.md#gatewayrefconsumerconfig) for the full field reference. @@ -194,7 +198,7 @@ an empty list in arbitrary startup YAML: |---|---|---|---| | Praxis `intelligent_route` with `overlay_file` | Grid's versioned `routing-overlay.json` ConfigMap | A valid versioned envelope whose `overlay.candidates` is `[]` | Praxis validates and atomically serves the new snapshot; malformed replacements retain last-known-good. Grid distributes an empty revision only after `GatewayRef.supportsEmptyOverlay: true`. | | Generated `GatewayRef.consumerConfig` | The same scoped versioned overlay; generated `praxis.yaml` supplies filter and endpoint plumbing | The same empty envelope | Candidate-only updates hot reload. Credential-bearing revisions are held until `enableProjectedCredentials` has been rolled out and `supportsProjectedCredentials: true` attests the filter and Secret mounts are active. Listener, endpoint/TLS, and filter-pipeline changes still require the consumer owner to reload or roll out its Praxis configuration. | -| Embedded `grid-gateway` `grid_site_route` filter | The operator-published `grid-serving--` ConfigMap | A valid serving config with `candidates: []` | The running gateway watches the projected serving file and atomically replaces its candidate snapshot and provider-hop allowlist. Malformed updates retain the previous snapshot. | +| Embedded `grid-gateway` `grid_site_route` filter | The operator-published `grid-serving--` ConfigMap | A valid serving config with `candidates: []` | The running gateway watches the projected serving file and atomically replaces its candidate snapshot and provider-hop allowlist. Malformed updates retain the previous snapshot. Provider-hop trust is declared separately with `GatewayRef.providerHopEndpoints`. | `GatewayRef.supportsEmptyOverlay` defaults to `false`. Set it to `true` only after every data-plane consumer of that gateway's overlay has been upgraded and @@ -205,10 +209,12 @@ compatibility boundary, not a successful withdrawal. The embedded gateway's filter chain and upstream cluster definitions remain in its startup Praxis configuration, but its changing candidate list is not a startup-only inline list: it comes from the watched Grid serving ConfigMap. -Provider-hop context is emitted only for clusters explicitly configured as -`mutual_tls`; the embedded gateway carries the Grid overlay's stable candidate -ID and generates a fresh hop request ID. Caller-supplied routing-context -headers are removed before forwarding. The provider gateway still authenticates +Its provider-hop allowlist comes from the separate +`GatewayRef.providerHopEndpoints` field, not from `consumerConfig`; each entry +must declare `mutual_tls` and a nonblank SNI matching the embedded gateway's +verified upstream configuration. The embedded gateway carries the Grid +overlay's stable candidate ID and generates a fresh hop request ID. +Caller-supplied routing-context headers are removed before forwarding. The provider gateway still authenticates the peer with mTLS before consuming that context. Static, manually configured `intelligent_route.candidates` remain a distinct diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 18ff6f3c9..f7fb39eba 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -286,6 +286,16 @@ never starts clocks or collects. A pass that would delete more than half the stubs, and more than 8, deletes none and logs a warning; a partition looks like mass departure. +### GatewayRef.providerHopEndpoints + +`spec.gatewayRefs[].providerHopEndpoints` independently configures the +provider-hop trust allowlist used by the embedded `grid-gateway`. Each entry +names a provider cluster and declares its verified mTLS mode and SNI. The +embedded gateway's startup upstream configuration must use that TLS identity. +This field is independent of `consumerConfig`; absent or disabled generated +consumer Praxis config cannot make embedded serving fail validation or change +its provider-hop allowlist. + ### GatewayRef.consumerConfig `spec.gatewayRefs[].consumerConfig` opts a gateway into operator-managed consumer @@ -361,8 +371,10 @@ are retained. In projected mode, the generated config uses an empty credential table and resolves the selected Secret reference at request time, failing closed when it cannot read it. -When `enabled: false` or `consumerConfig` is absent, this gateway behaves as -before — only the routing overlay `ConfigMap` is applied. +When `enabled: false` or `consumerConfig` is absent, no generated consumer +Praxis `ConfigMap` is applied. The routing overlay remains active, and the +embedded gateway's serving config is controlled separately by +`GatewayRef.providerHopEndpoints`. ## GridSite diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 430c1594b..71832fc5f 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -2484,12 +2484,7 @@ fn render_serving_text( .map_or(crate::crd::grid_network::DEFAULT_TLS_CERT_MOUNT_PATH, |cc| { cc.tls_cert_mount_path.as_str() }); - let provider_hop_clusters = gw_ref - .consumer_config - .as_ref() - .map(|config| consumer_config::provider_hop_clusters(&config.cluster_endpoints)) - .transpose()? - .unwrap_or_default(); + let provider_hop_clusters = serving_provider_hop_clusters(gw_ref)?; let inputs = ServingInputs { tls_mount, local_signals_addr: source.settings.local_signals_addr.as_deref(), @@ -2500,6 +2495,43 @@ fn render_serving_text( serving_config::to_text(&serving_config::render(overlay, members, &inputs)).map_err(OperatorError::Json) } +/// Resolve embedded-gateway provider hops from their dedicated GatewayRef +/// contract, not from the optional generated consumer Praxis config. +fn serving_provider_hop_clusters(gw_ref: &GatewayRef) -> Result, OperatorError> { + let mut clusters = BTreeSet::new(); + for endpoint in &gw_ref.provider_hop_endpoints { + if endpoint.cluster.trim().is_empty() { + return Err(OperatorError::InvalidResource( + "providerHopEndpoints cluster must not be blank".to_owned(), + )); + } + if !clusters.insert(endpoint.cluster.clone()) { + return Err(OperatorError::InvalidResource(format!( + "providerHopEndpoints contains duplicate cluster {:?}", + endpoint.cluster + ))); + } + if endpoint.transport.mode != TransportMode::MutualTls { + return Err(OperatorError::InvalidResource(format!( + "providerHopEndpoints cluster {:?} must use mutual_tls", + endpoint.cluster + ))); + } + if endpoint + .transport + .sni + .as_deref() + .is_none_or(|sni| sni.trim().is_empty()) + { + return Err(OperatorError::InvalidResource(format!( + "providerHopEndpoints cluster {:?} requires a nonblank SNI", + endpoint.cluster + ))); + } + } + Ok(clusters) +} + /// Apply the serving config `ConfigMap` when changed, returning when to retry a deferred write. async fn apply_serving_config( overlay: &routing_overlay::RoutingOverlay, @@ -5991,6 +6023,7 @@ fn parse_metrics_refresh_interval(value: &str) -> Result, + /// Opt-in configuration for operator-managed consumer Praxis config generation. /// /// When absent or `enabled: false`, this gateway behaves exactly as before — @@ -1125,6 +1134,19 @@ pub struct EndpointCaSecretRef { pub key: Option, } +/// Explicit verified-mTLS endpoint identity trusted for embedded provider-hop +/// context. This does not configure the gateway's upstream TLS connection; the +/// embedded gateway must separately use verified TLS with this SNI. +#[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ProviderHopEndpointConfig { + /// Cluster name used by the embedded gateway's `load_balancer` filter. + pub cluster: String, + + /// TLS mode and SNI configured for that embedded gateway upstream. + pub transport: EndpointTransport, +} + /// Endpoint configuration for one consumer `load_balancer` cluster. /// /// Maps an inference candidate cluster name to a reachable provider gateway @@ -2067,6 +2089,7 @@ mod tests { namespace: "ns".to_owned(), local_site_name: None, supports_empty_overlay: false, + provider_hop_endpoints: Vec::new(), consumer_config: None, }; let json = serde_json::to_value(&gw).unwrap_or_else(|_| std::process::abort()); @@ -2091,6 +2114,10 @@ mod tests { gateway_ref_properties.contains_key("consumerConfig"), "CRD schema must include consumerConfig field on GatewayRef" ); + assert!( + gateway_ref_properties.contains_key("providerHopEndpoints"), + "CRD schema must include independent providerHopEndpoints on GatewayRef" + ); let consumer_config_properties = gateway_ref_properties .get("consumerConfig") .and_then(|v| v.pointer("/properties")) diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index ee69440cd..eed80c5ac 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -815,9 +815,9 @@ fn render_provider_hop_clusters(cluster_endpoints: &[ClusterEndpointConfig]) -> Ok(format!(" provider_hop_clusters: [{values}]\n")) } -/// The explicit mTLS endpoint names allowed to receive provider-hop context. -/// Validate the endpoint identities here because this allowlist is also used by -/// the embedded Grid gateway, independently of generated Praxis config. +/// The explicit mTLS endpoint names allowed to receive provider-hop context in +/// generated consumer Praxis config. The embedded Grid gateway uses its +/// independent `GatewayRef.providerHopEndpoints` contract. #[expect( clippy::too_many_lines, reason = "this validation keeps the mTLS provider-hop boundary explicit" diff --git a/xtask/src/env/provider_traffic_qualification.rs b/xtask/src/env/provider_traffic_qualification.rs index eb3f68658..562b1e21f 100644 --- a/xtask/src/env/provider_traffic_qualification.rs +++ b/xtask/src/env/provider_traffic_qualification.rs @@ -2949,15 +2949,28 @@ fn patch_grid_network_for_serving( /// Build the run-owned `GatewayRef` with the explicit mTLS provider-hop inventory. fn embedded_serving_gateway_ref(endpoints: &[serde_json::Value]) -> serde_json::Value { + let provider_hop_endpoints: Vec<_> = endpoints + .iter() + .map(|endpoint| { + serde_json::json!({ + "cluster": endpoint.get("cluster"), + "transport": endpoint.get("transport") + }) + }) + .collect(); serde_json::json!({ "name": GRID_SERVING_GATEWAY, "namespace": GRID_SYSTEM_NS, "localSiteName": CONSUMER_SITE, "supportsEmptyOverlay": true, + "providerHopEndpoints": provider_hop_endpoints, "consumerConfig": { "enabled": false, "tlsCertMountPath": "/etc/praxis/tls", - "clusterEndpoints": endpoints + "clusterEndpoints": [ + { "cluster": "", "address": "", "transport": null }, + { "cluster": "", "address": "", "transport": null } + ] } }) } @@ -5562,7 +5575,7 @@ mod tests { } #[test] - fn embedded_gateway_reference_declares_mtls_provider_hops() { + fn embedded_gateway_declares_mtls_provider_hops_separately_from_disabled_consumer_config() { let endpoints = PROVIDER_RESOURCES .iter() .map(|(site, resource)| { @@ -5581,7 +5594,7 @@ mod tests { Some(false) ); let rendered_endpoints = gateway - .pointer("/consumerConfig/clusterEndpoints") + .pointer("/providerHopEndpoints") .and_then(serde_json::Value::as_array) .unwrap_or_else(|| std::process::abort()); assert_eq!(rendered_endpoints.len(), PROVIDER_RESOURCES.len()); @@ -5592,6 +5605,12 @@ mod tests { .and_then(serde_json::Value::as_str) .is_some() })); + assert_eq!( + gateway + .pointer("/consumerConfig/clusterEndpoints/0/cluster") + .and_then(serde_json::Value::as_str), + Some("") + ); } #[test] From 3a47b0a9c1e9445a168d431a7d4f8922c724dc49 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 3 Oct 2026 00:33:59 +0000 Subject: [PATCH 03/10] fix(routing): enforce no-route serving across gateways Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 9 +- deploy/crds/gridnetwork.yaml | 9 +- gateway/ai-grid-filters/src/control.rs | 13 +- gateway/ai-grid-filters/src/route.rs | 15 + gateway/ai-grid-filters/src/serving.rs | 18 +- gateway/src/main.rs | 68 +- operator/src/controller/grid_network.rs | 43 +- operator/src/crd/grid_network.rs | 9 +- operator/src/resources/serving_config.rs | 9 + .../src/env/provider_traffic_qualification.rs | 830 +++++++++++++++++- .../src/env/static_weighted_qualification.rs | 90 +- 11 files changed, 983 insertions(+), 130 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index 2ec400c38..9bcdacf85 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -455,13 +455,16 @@ spec: serving config. This allowlist is independent of `consumerConfig`, so a missing or disabled generated Praxis config cannot alter embedded serving behavior. Every entry must declare `mutual_tls` and a nonblank - SNI; the embedded gateway's upstream configuration must use that - verified TLS identity. + SNI. The embedded gateway compares each declaration against the + loaded Praxis load-balancer's verified mTLS backend before it sends + provider-hop context. Changing Praxis backend transport requires a + gateway restart while `gridServing` is enabled. items: description: |- Explicit verified-mTLS endpoint identity trusted for embedded provider-hop context. This does not configure the gateway's upstream TLS connection; the - embedded gateway must separately use verified TLS with this SNI. + embedded gateway must separately use verified TLS with this SNI; the + gateway checks the two configurations before serving hop traffic. properties: cluster: description: Cluster name used by the embedded gateway's `load_balancer` filter. diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 20232b449..6f40ecf6f 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -447,13 +447,16 @@ spec: serving config. This allowlist is independent of `consumerConfig`, so a missing or disabled generated Praxis config cannot alter embedded serving behavior. Every entry must declare `mutual_tls` and a nonblank - SNI; the embedded gateway's upstream configuration must use that - verified TLS identity. + SNI. The embedded gateway compares each declaration against the + loaded Praxis load-balancer's verified mTLS backend before it sends + provider-hop context. Changing Praxis backend transport requires a + gateway restart while `gridServing` is enabled. items: description: |- Explicit verified-mTLS endpoint identity trusted for embedded provider-hop context. This does not configure the gateway's upstream TLS connection; the - embedded gateway must separately use verified TLS with this SNI. + embedded gateway must separately use verified TLS with this SNI; the + gateway checks the two configurations before serving hop traffic. properties: cluster: description: Cluster name used by the embedded gateway's `load_balancer` filter. diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index 138992f96..c7fe132df 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -1,7 +1,7 @@ //! Applies a grid serving config: the candidate topology and the peer pollers. use std::{ - collections::{BTreeSet, HashMap}, + collections::{BTreeMap, BTreeSet, HashMap}, path::PathBuf, sync::{ Arc, Mutex, OnceLock, PoisonError, Weak, @@ -229,6 +229,16 @@ pub(crate) struct Control { impl Control { /// Build the control plane for `config` without starting any poller. pub(crate) fn new(config: &GridServingConfig, start: StartPeer) -> Result { + Self::new_with_backend_tls(config, start, BTreeMap::new()) + } + + /// Build with the verified backend identities that serving reloads must preserve. + pub(crate) fn new_with_backend_tls( + config: &GridServingConfig, + start: StartPeer, + backend_tls: BTreeMap, + ) -> Result { + validate_provider_hop_binding(config, &backend_tls)?; let topology = Topology::from_config(config)?; // Cold start: config order until the first poll re-orders it by live load. let mut gauged = Gauged::new(); @@ -829,6 +839,7 @@ mod tests { load_window_ms: 30_000, candidates: sites.iter().map(|site| candidate(site)).collect(), provider_hop_clusters: Vec::new(), + provider_hop_sni: BTreeMap::new(), peers: sites.iter().map(|site| peer(site)).collect(), } } diff --git a/gateway/ai-grid-filters/src/route.rs b/gateway/ai-grid-filters/src/route.rs index 267e52204..acaa4da35 100644 --- a/gateway/ai-grid-filters/src/route.rs +++ b/gateway/ai-grid-filters/src/route.rs @@ -1834,4 +1834,19 @@ mod tests { assert!((0..10_000).all(|turn| (0.0..1.0).contains(&unit(turn, SALTS.1)))); assert_ne!(random_seed(), random_seed()); } + + #[test] + fn empty_snapshot_rejects_even_with_a_preselected_cluster() { + let empty = RouteSnapshot::from_static(Vec::new(), Arc::from("local")); + assert!(matches!(route_decision(&empty, "llama", true), RouteDecision::NoRoute)); + + let active = RouteSnapshot::from_static( + one("llama", "east", "pool-a", AdmissionState::NewAndExisting), + Arc::from("local"), + ); + assert!(matches!( + route_decision(&active, "llama", true), + RouteDecision::KeepEarlier + )); + } } diff --git a/gateway/ai-grid-filters/src/serving.rs b/gateway/ai-grid-filters/src/serving.rs index e849772bf..fcdb946c2 100644 --- a/gateway/ai-grid-filters/src/serving.rs +++ b/gateway/ai-grid-filters/src/serving.rs @@ -7,6 +7,7 @@ //! resolved order. A watch on the config file applies the operator's rewrites. use std::{ + collections::{BTreeMap, BTreeSet}, fs, path::PathBuf, sync::{Arc, Mutex, PoisonError}, @@ -64,6 +65,10 @@ pub struct GridServingConfig { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub provider_hop_clusters: Vec, + /// TLS SNI declared for each authenticated provider hop. + #[serde(default)] + pub provider_hop_sni: BTreeMap, + /// Peers to poll for live load. pub peers: Vec, } @@ -322,7 +327,7 @@ pub fn spawn_grid_routing(config: &GridServingConfig) -> Result FilterError { format!("grid: spawning poller for {}: {error}", peer.site).into() }) }); - start_runtime(config, start) + start_runtime_with_backend_tls(config, start, backend_tls) } /// Build the runtime over `start`, the peer poller constructor. @@ -330,7 +335,16 @@ pub(crate) fn start_runtime( config: &GridServingConfig, start: crate::control::StartPeer, ) -> Result { - let mut control = Control::new(config, start)?; + start_runtime_with_backend_tls(config, start, BTreeMap::new()) +} + +/// Build the runtime with TLS identities from the loaded Praxis backends. +fn start_runtime_with_backend_tls( + config: &GridServingConfig, + start: crate::control::StartPeer, + backend_tls: BTreeMap, +) -> Result { + let mut control = Control::new_with_backend_tls(config, start, backend_tls)?; control.apply(config)?; let health_tick = tick_health(control.health(), control.refresh(), control.store()) .map_err(|error| -> FilterError { format!("grid: spawning the health tick: {error}").into() })?; diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 1ffc7c4bf..c06def60b 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -15,6 +15,7 @@ use std::{ffi::OsStr, process::ExitCode}; use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; +use serde::Deserialize; use tracing::info; mod metrics_listener; @@ -110,7 +111,7 @@ fn main() -> ExitCode { // bound until the server returns. let grid_runtime = match std::env::var("GRID_SERVING_CONFIG") .ok() - .map(|path| start_grid_routing(&path, &mut registry)) + .map(|path| start_grid_routing(&path, &config, &mut registry)) { Some(Err(err)) => return praxis::report_fatal(&err, log_output), Some(Ok(runtime)) => Some(runtime), @@ -212,6 +213,7 @@ fn start_metrics_listener(config: &Config) -> Result<(), String> { /// registering the filters. fn start_grid_routing( path: &str, + praxis_config: &Config, registry: &mut praxis_filter::FilterRegistry, ) -> Result { let config = ai_grid_filters::load_serving_config(path)?; @@ -233,6 +235,70 @@ fn start_grid_routing( /// How often the grid serving config file is re-read. const SERVING_RELOAD_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5); +/// Minimal view of a load-balancer cluster for provider-hop trust validation. +#[derive(Deserialize)] +struct LoadBalancerBackends { + clusters: Vec, +} + +#[derive(Deserialize)] +struct BackendCluster { + name: String, + tls: Option, +} + +#[derive(Deserialize)] +struct BackendTls { + sni: String, + verify: bool, + ca: Option, + client_cert: Option, +} + +#[derive(Deserialize)] +struct BackendCa { + ca_path: String, +} + +#[derive(Deserialize)] +struct BackendClientCert { + cert_path: String, + key_path: String, +} + +/// Only verified mutual-TLS backends in Grid-enabled chains can carry hop context. +#[expect(clippy::too_many_lines, reason = "keeps the backend transport checks together")] +fn provider_hop_backends(config: &Config) -> Result, praxis_filter::FilterError> { + let mut backends = BTreeMap::new(); + for chain in &config.filter_chains { + if !chain.filters.iter().any(|filter| filter.filter_type == "grid_site_route") { + continue; + } + for filter in chain.filters.iter().filter(|filter| filter.filter_type == "load_balancer") { + let parsed: LoadBalancerBackends = serde_yaml::from_value(filter.config.clone()) + .map_err(|error| -> praxis_filter::FilterError { + format!("grid: parsing load_balancer backends: {error}").into() + })?; + for backend in parsed.clusters { + let Some(tls) = backend.tls else { continue }; + if !tls.verify + || tls.sni.trim().is_empty() + || tls.ca.as_ref().is_none_or(|ca| ca.ca_path.trim().is_empty()) + || tls.client_cert.as_ref().is_none_or(|cert| { + cert.cert_path.trim().is_empty() || cert.key_path.trim().is_empty() + }) + { + continue; + } + if backends.insert(backend.name.clone(), tls.sni).is_some() { + return Err(format!("grid: duplicate verified backend {:?}", backend.name).into()); + } + } + } + } + Ok(backends) +} + /// Usage line for a malformed command line. const USAGE: &str = "usage: grid-gateway [--config | -c | ]"; diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 71832fc5f..14aef76a8 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -2343,7 +2343,7 @@ async fn reconcile_routing_overlay_inner( observed_generation, }); - // The gateway reads it only at start, so a failure never blocks the overlay. + // The embedded gateway watches valid serving revisions independently. if let Some(source) = serving { match apply_serving_config(&overlay, source, network_name, gw_ref, client).await { Ok(retry) => serving_retry = serving_retry.into_iter().chain(retry).min(), @@ -2484,7 +2484,8 @@ fn render_serving_text( .map_or(crate::crd::grid_network::DEFAULT_TLS_CERT_MOUNT_PATH, |cc| { cc.tls_cert_mount_path.as_str() }); - let provider_hop_clusters = serving_provider_hop_clusters(gw_ref)?; + let provider_hop_sni = serving_provider_hop_sni_for_overlay(overlay, gw_ref)?; + let provider_hop_clusters = provider_hop_sni.keys().cloned().collect(); let inputs = ServingInputs { tls_mount, local_signals_addr: source.settings.local_signals_addr.as_deref(), @@ -2495,8 +2496,38 @@ fn render_serving_text( serving_config::to_text(&serving_config::render(overlay, members, &inputs)).map_err(OperatorError::Json) } -/// Resolve embedded-gateway provider hops from their dedicated GatewayRef +/// No candidate can receive hop context after an authoritative withdrawal. +fn serving_provider_hop_sni_for_overlay( + overlay: &routing_overlay::RoutingOverlay, + gw_ref: &GatewayRef, +) -> Result, OperatorError> { + if overlay.candidates.is_empty() { + if let Err(error) = serving_provider_hop_clusters(gw_ref) { + tracing::warn!(gateway = %gw_ref.name, %error, + "invalid provider-hop declaration ignored for empty serving revision"); + } + Ok(std::collections::BTreeMap::new()) + } else { + serving_provider_hop_clusters(gw_ref)?; + Ok(gw_ref + .provider_hop_endpoints + .iter() + .map(|endpoint| { + ( + endpoint.cluster.clone(), + endpoint.transport.sni.clone().unwrap_or_default(), + ) + }) + .collect()) + } +} + +/// Resolve embedded-gateway provider hops from their dedicated `GatewayRef` /// contract, not from the optional generated consumer Praxis config. +#[expect( + clippy::too_many_lines, + reason = "each provider-hop declaration is checked before routing" +)] fn serving_provider_hop_clusters(gw_ref: &GatewayRef) -> Result, OperatorError> { let mut clusters = BTreeSet::new(); for endpoint in &gw_ref.provider_hop_endpoints { @@ -6023,7 +6054,10 @@ fn parse_metrics_refresh_interval(value: &str) -> Result SocketAddr { value.parse().unwrap_or_else(|_| std::process::abort()) diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 31d41e82b..74d627565 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -758,8 +758,10 @@ pub struct GatewayRef { /// serving config. This allowlist is independent of `consumerConfig`, so a /// missing or disabled generated Praxis config cannot alter embedded /// serving behavior. Every entry must declare `mutual_tls` and a nonblank - /// SNI; the embedded gateway's upstream configuration must use that - /// verified TLS identity. + /// SNI. The embedded gateway compares each declaration against the + /// loaded Praxis load-balancer's verified mTLS backend before it sends + /// provider-hop context. Changing Praxis backend transport requires a + /// gateway restart while `gridServing` is enabled. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub provider_hop_endpoints: Vec, @@ -1136,7 +1138,8 @@ pub struct EndpointCaSecretRef { /// Explicit verified-mTLS endpoint identity trusted for embedded provider-hop /// context. This does not configure the gateway's upstream TLS connection; the -/// embedded gateway must separately use verified TLS with this SNI. +/// embedded gateway must separately use verified TLS with this SNI; the +/// gateway checks the two configurations before serving hop traffic. #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct ProviderHopEndpointConfig { diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index 36174e278..d8f55dd07 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -76,6 +76,9 @@ pub(crate) struct ServingConfig { /// Explicit mTLS provider gateways authorized to receive hop context. #[serde(skip_serializing_if = "Vec::is_empty")] pub(crate) provider_hop_clusters: Vec, + /// Expected verified TLS SNI for each provider-hop cluster. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub(crate) provider_hop_sni: BTreeMap, /// Sorted by site. pub(crate) peers: Vec, } @@ -181,6 +184,7 @@ where load_window_ms: load_window_ms(inputs.scrape_interval), candidates, provider_hop_clusters: inputs.provider_hop_clusters.iter().cloned().collect(), + provider_hop_sni: inputs.provider_hop_sni.clone(), peers, } } @@ -466,6 +470,8 @@ mod tests { static NO_PINS: BTreeMap> = BTreeMap::new(); /// No authenticated provider gateway in this fixture. static NO_PROVIDER_HOPS: BTreeSet = BTreeSet::new(); + /// No declared provider-hop identities in the default fixture. + static NO_PROVIDER_HOP_SNI: BTreeMap = BTreeMap::new(); const INPUTS: ServingInputs<'static> = ServingInputs { tls_mount: "/etc/praxis/tls", @@ -761,12 +767,15 @@ mod tests { let mut candidate = cand("llama", "site-b", "provider-b", None); candidate.stable_id = Some("257a9450".to_owned()); let hops = BTreeSet::from(["provider-b".to_owned()]); + let hop_sni = BTreeMap::from([("provider-b".to_owned(), "provider-b.example".to_owned())]); let inputs = ServingInputs { provider_hop_clusters: &hops, + provider_hop_sni: &hop_sni, ..INPUTS }; let config = render(&overlay(vec![candidate]), [], &inputs); assert_eq!(config.provider_hop_clusters, ["provider-b"]); + assert_eq!(config.provider_hop_sni, hop_sni); assert_eq!(config.candidates[0].stable_id.as_deref(), Some("257a9450")); let json = to_text(&config).expect("JSON"); assert!(json.contains("\"provider_hop_clusters\": [\n \"provider-b\"")); diff --git a/xtask/src/env/provider_traffic_qualification.rs b/xtask/src/env/provider_traffic_qualification.rs index 562b1e21f..1f238303c 100644 --- a/xtask/src/env/provider_traffic_qualification.rs +++ b/xtask/src/env/provider_traffic_qualification.rs @@ -482,13 +482,80 @@ fn curl_pod_overrides(pod_name: &str, curl_args: &[&str]) -> String { .to_string() } -/// Run an ephemeral curl pod with restricted `PodSecurity` context. -fn run_curl_probe(context: &str, pod_name: &str, curl_args: &[&str]) -> Result { +/// `kubectl run`'s attached stream can lose output from a short-lived curl +/// container. Read logs only after Kubernetes reports a terminated container. +struct ProbeExitStatus { + code: i32, +} + +impl ProbeExitStatus { + fn success(&self) -> bool { + self.code == 0 + } +} + +struct CurlProbeOutput { + status: ProbeExitStatus, + stdout: Vec, + stderr: Vec, +} + +/// Delete exactly the temporary probe Pod created by `run_curl_probe`. +struct ProbePodCleanup<'a> { + context: &'a str, + name: &'a str, +} + +impl Drop for ProbePodCleanup<'_> { + fn drop(&mut self) { + drop( + Command::new("kubectl") + .args([ + "--context", + self.context, + "-n", + GRID_SYSTEM_NS, + "delete", + "pod", + self.name, + "--ignore-not-found=true", + "--wait=false", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(), + ); + } +} + +/// Return the terminal curl-container exit code, if Kubernetes has observed it. +fn probe_pod_exit_code(pod: &serde_json::Value) -> Option { + pod.pointer("/status/containerStatuses") + .and_then(serde_json::Value::as_array) + .and_then(|statuses| statuses.first()) + .and_then(|status| status.pointer("/state/terminated/exitCode")) + .and_then(serde_json::Value::as_i64) + .and_then(|code| i32::try_from(code).ok()) + .or_else( + || match pod.pointer("/status/phase").and_then(serde_json::Value::as_str) { + Some("Succeeded") => Some(0), + Some("Failed") => Some(1), + _ => None, + }, + ) +} + +/// Run a restricted curl Pod, collect its completed logs, and then remove it. +fn run_curl_probe( + context: &str, + pod_name: &str, + curl_args: &[&str], +) -> Result> { let sequence = PROBE_SEQUENCE.fetch_add(1, Ordering::Relaxed); let prefix = pod_name.get(..pod_name.len().min(40)).unwrap_or(pod_name); let unique_pod_name = format!("{prefix}-{sequence}"); let overrides = curl_pod_overrides(&unique_pod_name, curl_args); - Command::new("kubectl") + let created = Command::new("kubectl") .args([ "run", &unique_pod_name, @@ -497,13 +564,96 @@ fn run_curl_probe(context: &str, pod_name: &str, curl_args: &[&str]) -> Result>() + .join("\n"); + if diagnostics.is_empty() { + format!("curl container exited with status {code}") + } else { + diagnostics + } + .into_bytes() + }; + return Ok(CurlProbeOutput { + status: ProbeExitStatus { code }, + stdout: logs.stdout, + stderr, + }); + } + if pod.pointer("/status/phase").and_then(serde_json::Value::as_str) == Some("Failed") { + let reason = pod + .pointer("/status/reason") + .and_then(serde_json::Value::as_str) + .unwrap_or("PodFailed"); + let message = pod + .pointer("/status/message") + .and_then(serde_json::Value::as_str) + .unwrap_or("curl probe Pod failed before the container reported an exit code"); + return Err(format!("curl probe Pod failed: {reason}: {}", safe_truncate_str(message, 500)).into()); + } + } else if !String::from_utf8_lossy(&observed.stderr).contains("NotFound") { + return Err(format!( + "could not observe curl probe Pod {unique_pod_name}: {}", + safe_truncate_str(String::from_utf8_lossy(&observed.stderr).trim(), 500) + ) + .into()); + } + if Instant::now() >= deadline { + return Err(format!("timed out waiting for curl probe Pod {unique_pod_name} to finish").into()); + } + std::thread::park_timeout(Duration::from_millis(200)); + } } /// Run an ephemeral curl pod with additional kubectl flags (e.g. `--labels`). @@ -515,6 +665,35 @@ fn response_header(output: &[u8], name: &str) -> Option { }) } +/// Extract the final HTTP status line without retaining a response body. +fn response_status_line(output: &[u8]) -> Option { + String::from_utf8_lossy(output) + .lines() + .rev() + .find(|line| line.starts_with("HTTP/")) + .map(|line| safe_truncate_str(line.trim(), 120)) +} + +/// Return response header names only, for sanitized probe diagnostics. +fn response_header_names(output: &[u8]) -> Vec { + let mut names = BTreeSet::new(); + let mut in_headers = false; + for line in String::from_utf8_lossy(output).lines() { + if line.starts_with("HTTP/") { + in_headers = true; + continue; + } + if line.is_empty() { + in_headers = false; + continue; + } + if in_headers && let Some((name, _)) = line.split_once(':') { + names.insert(name.trim().to_ascii_lowercase()); + } + } + names.into_iter().collect() +} + /// Parse curl's explicit HTTP status marker, rejecting transport failures (`000`). fn curl_http_status(output: &[u8]) -> Result { let response = String::from_utf8_lossy(output); @@ -1032,8 +1211,8 @@ fn assert_overlay_acceptance() -> AssertionResult { .parse::() .is_ok_and(|n| n > 0); - // Step 4: A valid request proves that the accepted overlay is serving. - // Praxis health endpoints are exposed only on the pod-local admin listener. + // Step 4: A request proves routing works; the exact revision gate below + // distinguishes the current overlay from an older last-known-good one. let routing_output = run_curl_probe( &context, &format!("overlay-routing-{cluster}"), @@ -1056,7 +1235,10 @@ fn assert_overlay_acceptance() -> AssertionResult { let routing_ok = routing_output.status.success(); - let overlay_accepted = overlay_exists && has_data && gateway_ready && routing_ok; + let current_overlay = read_cluster_overlay(cluster)?; + let revision_accepted = current_overlay.semantic_revision != "unknown" + && wait_for_consumer_gateway_revision(¤t_overlay.semantic_revision).is_ok(); + let overlay_accepted = overlay_exists && has_data && gateway_ready && routing_ok && revision_accepted; if !overlay_accepted { all_accepted = false; } @@ -1074,6 +1256,14 @@ fn assert_overlay_acceptance() -> AssertionResult { format!("{cluster}_overlay_routing_ok"), serde_json::Value::Bool(routing_ok), ); + observed_facts.insert( + format!("{cluster}_accepted_serving_revision_matches"), + serde_json::Value::Bool(revision_accepted), + ); + observed_facts.insert( + format!("{cluster}_overlay_revision"), + serde_json::Value::String(current_overlay.semantic_revision), + ); observed_facts.insert( format!("{cluster}_resource_version"), serde_json::Value::String(resource_version), @@ -1087,7 +1277,7 @@ fn assert_overlay_acceptance() -> AssertionResult { if all_accepted { Ok(proof_success( - "Consumer entrypoint overlay ConfigMap exists with data, gateway ready, and routing passes", + "Consumer entrypoint accepted and serves the current overlay revision, with a ready gateway and successful routing", observed_facts, start.elapsed(), )) @@ -2638,6 +2828,81 @@ fn enable_generated_consumer_config() -> Result<(), Box> Ok(()) } +/// Change only the run-owned Provider A consumer endpoint transport and return its previous mode. +fn set_generated_consumer_provider_a_transport_mode(mode: &str) -> Result> { + let context = cluster_context("provider-a"); + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + GRID_NETWORK_NAME, + "-o", + "json", + ]) + .output()?; + if !output.status.success() { + return Err("could not read provider-a GridNetwork before changing consumerConfig".into()); + } + let network: serde_json::Value = serde_json::from_slice(&output.stdout)?; + let gateways = network + .pointer("/spec/gatewayRefs") + .and_then(serde_json::Value::as_array) + .ok_or("provider-a GridNetwork has no gatewayRefs array")?; + let gateway_index = gateways + .iter() + .position(|gateway| { + gateway.get("name").and_then(serde_json::Value::as_str) == Some("consumer-gateway") + && gateway.get("namespace").and_then(serde_json::Value::as_str) == Some(GRID_SYSTEM_NS) + }) + .ok_or("provider-a GridNetwork does not declare its run-owned consumer-gateway")?; + let endpoints = gateways + .get(gateway_index) + .and_then(|gateway| gateway.pointer("/consumerConfig/clusterEndpoints")) + .and_then(serde_json::Value::as_array) + .ok_or("run-owned consumerConfig has no clusterEndpoints array")?; + let endpoint_index = endpoints + .iter() + .position(|endpoint| { + endpoint.get("cluster").and_then(serde_json::Value::as_str) == Some("vcr-provider-a-provider") + }) + .ok_or("run-owned consumerConfig has no Provider A endpoint")?; + let previous_mode = endpoints + .get(endpoint_index) + .and_then(|endpoint| endpoint.pointer("/transport/mode")) + .and_then(serde_json::Value::as_str) + .ok_or("Provider A consumer endpoint has no transport mode")? + .to_owned(); + let path = + format!("/spec/gatewayRefs/{gateway_index}/consumerConfig/clusterEndpoints/{endpoint_index}/transport/mode"); + let patch = serde_json::json!([{"op":"replace","path":path,"value":mode}]); + let patched = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "patch", + "gridnetwork", + GRID_NETWORK_NAME, + "--type=json", + "--patch", + &patch.to_string(), + ]) + .output()?; + if !patched.status.success() { + return Err(format!( + "could not set run-owned Provider A consumer endpoint transport: {}", + safe_truncate_str(String::from_utf8_lossy(&patched.stderr).trim(), 800) + ) + .into()); + } + Ok(previous_mode) +} + /// Wait for a generation-current Rendered status and validate its live config. #[expect( clippy::too_many_lines, @@ -2717,6 +2982,66 @@ fn wait_for_generated_consumer_config() -> Result Result> { + const TIMEOUT: Duration = Duration::from_secs(180); + let context = cluster_context("provider-a"); + let deadline = Instant::now() + TIMEOUT; + let mut last_state = String::from("not observed"); + loop { + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "get", + "gridnetwork", + GRID_NETWORK_NAME, + "-o", + "json", + ]) + .output()?; + if output.status.success() { + let network: serde_json::Value = serde_json::from_slice(&output.stdout)?; + let generation = network + .pointer("/metadata/generation") + .and_then(serde_json::Value::as_i64); + let statuses = network + .pointer("/status/consumerConfigStatus") + .and_then(serde_json::Value::as_array); + let current = statuses.and_then(|items| { + items.iter().find(|item| { + item.get("gatewayName").and_then(serde_json::Value::as_str) == Some("consumer-gateway") + && item.get("namespace").and_then(serde_json::Value::as_str) == Some(GRID_SYSTEM_NS) + }) + }); + if let Some(current) = current { + let observed = current.get("observedGeneration").and_then(serde_json::Value::as_i64); + let phase = current + .get("phase") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + let reason = current + .get("reason") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + last_state = + format!("phase={phase}, reason={reason}, observed={observed:?}, generation={generation:?}"); + if phase == "Error" && reason == expected_reason && observed == generation { + return Ok(current.clone()); + } + } + } + if Instant::now() >= deadline { + return Err(format!("timed out waiting for generation-current consumer config error: {last_state}").into()); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} + /// Check that generated YAML is dynamic, scoped, and contains the complete /// endpoint inventory rather than startup-only candidates. #[expect( @@ -2947,6 +3272,78 @@ fn patch_grid_network_for_serving( Ok(()) } +/// Replace only the run-owned embedded gateway's provider-hop declaration and +/// return its exact prior value for unconditional restoration. +fn patch_embedded_provider_hop_endpoints( + endpoints: Option<&serde_json::Value>, +) -> Result> { + let context = cluster_context(CONSUMER_SITE); + let network = kubectl_get_json(&context, &format!("gridnetwork/{GRID_NETWORK_NAME}"))?; + let mut gateway_refs = network + .pointer("/spec/gatewayRefs") + .and_then(serde_json::Value::as_array) + .cloned() + .ok_or("GridNetwork has no gatewayRefs array")?; + let index = gateway_refs + .iter() + .position(|gateway| gateway.get("name").and_then(serde_json::Value::as_str) == Some(GRID_SERVING_GATEWAY)) + .ok_or("GridNetwork has no run-owned embedded gateway reference")?; + let gateway = gateway_refs + .get_mut(index) + .ok_or("embedded gateway reference index is out of range")?; + let previous = gateway + .get("providerHopEndpoints") + .cloned() + .unwrap_or(serde_json::Value::Null); + if let Some(endpoints) = endpoints { + gateway["providerHopEndpoints"] = endpoints.clone(); + } else if let Some(object) = gateway.as_object_mut() { + object.remove("providerHopEndpoints"); + } + kubectl_patch_merge( + &context, + &format!("gridnetwork/{GRID_NETWORK_NAME}"), + &serde_json::json!({"spec": {"gatewayRefs": gateway_refs}}), + )?; + Ok(previous) +} + +/// Wait for the controller's warning that invalid optional hop metadata was +/// ignored while publishing an empty serving revision. +fn wait_for_empty_overlay_invalid_hop_warning(timeout: Duration) -> Result> { + let context = cluster_context(CONSUMER_SITE); + let deadline = Instant::now() + timeout; + loop { + let output = Command::new("kubectl") + .args([ + "--context", + &context, + "-n", + GRID_SYSTEM_NS, + "logs", + "deployment/grid-operator", + "--all-containers=true", + "--since=5m", + ]) + .output()?; + if output.status.success() { + let logs = String::from_utf8_lossy(&output.stdout); + if let Some(line) = logs + .lines() + .find(|line| line.contains("invalid provider-hop declaration ignored for empty serving revision")) + { + return Ok(safe_truncate_str(line, 1_000)); + } + } + if Instant::now() >= deadline { + return Err( + "operator did not log that invalid provider-hop metadata was ignored for the empty revision".into(), + ); + } + std::thread::park_timeout(Duration::from_secs(2)); + } +} + /// Build the run-owned `GatewayRef` with the explicit mTLS provider-hop inventory. fn embedded_serving_gateway_ref(endpoints: &[serde_json::Value]) -> serde_json::Value { let provider_hop_endpoints: Vec<_> = endpoints @@ -3001,6 +3398,26 @@ fn require_serving_provider_hops(snapshot: &serde_json::Value) -> Result<(), Box Ok(()) } +/// An authoritative empty serving revision must not carry provider-hop trust. +fn require_empty_serving_provider_hops(snapshot: &serde_json::Value) -> Result<(), Box> { + let serving = snapshot + .get("serving_config") + .ok_or("embedded serving snapshot has no serving_config")?; + if serving + .get("provider_hop_clusters") + .is_some_and(|value| !value.as_array().is_some_and(Vec::is_empty)) + { + return Err("empty embedded serving snapshot retained or malformed provider_hop_clusters".into()); + } + if serving + .get("provider_hop_sni") + .is_some_and(|value| !value.as_object().is_some_and(serde_json::Map::is_empty)) + { + return Err("empty embedded serving snapshot retained or malformed provider_hop_sni".into()); + } + Ok(()) +} + /// Return public site labels used by the provider-attribution response header. fn expected_provider_attributions() -> BTreeSet { PROVIDER_RESOURCES.iter().map(|(site, _)| (*site).to_owned()).collect() @@ -3895,6 +4312,7 @@ fn assert_provider_gateway_round_robin() -> AssertionResult { let mut counts: BTreeMap = BTreeMap::new(); let mut sequence = Vec::new(); let mut failures = Vec::new(); + let mut response_diagnostics = Vec::new(); let mut overlay_changes = Vec::new(); let readiness = match wait_for_round_robin_readiness() { @@ -3917,6 +4335,10 @@ fn assert_provider_gateway_round_robin() -> AssertionResult { )); }, }; + let gateway_posts_before: BTreeMap = PROVIDER_RESOURCES + .iter() + .map(|(site, _)| Ok(((*site).to_owned(), read_provider_gateway_post_count_cold(site)?))) + .collect::>>()?; for request_number in 1..=60 { if let Ok(current_overlay) = read_cluster_overlay("provider-a") @@ -3939,6 +4361,8 @@ fn assert_provider_gateway_round_robin() -> AssertionResult { "--include", "--silent", "--show-error", + "--write-out", + &format!("\n{CURL_HTTP_STATUS_MARKER}%{{http_code}}\n"), "--header", "Content-Type: application/json", "--header", @@ -3959,12 +4383,30 @@ fn assert_provider_gateway_round_robin() -> AssertionResult { sequence.push(provider); } else { failures.push(format!("{request_label}: provider attribution header missing")); + response_diagnostics.push(serde_json::json!({ + "request": request_label, + "curl_exit_code": output.status.code, + "http_status": curl_http_status(&output.stdout).ok(), + "http_status_line": response_status_line(&output.stdout), + "response_header_names": response_header_names(&output.stdout), + "response_bytes": output.stdout.len(), + })); } }, - Ok(output) => failures.push(format!( - "{request_label}: HTTP probe failed: {}", - String::from_utf8_lossy(&output.stderr).trim() - )), + Ok(output) => { + failures.push(format!( + "{request_label}: HTTP probe failed: {}", + safe_truncate_str(String::from_utf8_lossy(&output.stderr).trim(), 300) + )); + response_diagnostics.push(serde_json::json!({ + "request": request_label, + "curl_exit_code": output.status.code, + "http_status": curl_http_status(&output.stdout).ok(), + "http_status_line": response_status_line(&output.stdout), + "response_header_names": response_header_names(&output.stdout), + "response_bytes": output.stdout.len(), + })); + }, Err(error) => failures.push(format!("{request_label}: probe execution failed: {error}")), } } @@ -3985,8 +4427,20 @@ fn assert_provider_gateway_round_robin() -> AssertionResult { }) }); let balanced = exact_counts && repeating_cycle && failures.is_empty(); + let gateway_posts_after: BTreeMap = PROVIDER_RESOURCES + .iter() + .map(|(site, _)| Ok(((*site).to_owned(), read_provider_gateway_post_count(site)?))) + .collect::>>()?; let mut facts = BTreeMap::new(); + facts.insert( + "provider_gateway_posts_before".to_owned(), + serde_json::json!(gateway_posts_before), + ); + facts.insert( + "provider_gateway_posts_after".to_owned(), + serde_json::json!(gateway_posts_after), + ); facts.insert("request_count".to_owned(), serde_json::json!(sequence.len())); facts.insert("provider_counts".to_owned(), serde_json::json!(counts)); facts.insert("ordered_provider_sequence".to_owned(), serde_json::json!(sequence)); @@ -3997,6 +4451,10 @@ fn assert_provider_gateway_round_robin() -> AssertionResult { serde_json::json!(repeating_cycle), ); facts.insert("failures".to_owned(), serde_json::json!(failures)); + facts.insert( + "response_diagnostics".to_owned(), + serde_json::json!(response_diagnostics), + ); facts.insert("selection_policy".to_owned(), serde_json::json!("roundRobin")); facts.insert("scoring_strategy".to_owned(), serde_json::json!("noMetrics")); facts.insert("readiness".to_owned(), serde_json::json!(readiness)); @@ -4222,16 +4680,15 @@ fn send_lifecycle_request(session_id: Option<&str>) -> Result Result> { +fn read_workload_metrics(site: &str, resource: &str, remote_port: u16) -> Result> { let context = cluster_context(site); - let service = format!("service/vcr-inference-{site}"); let local_port = find_local_tcp_port()?; - let mapping = format!("{local_port}:8000"); + let mapping = format!("{local_port}:{remote_port}"); let child = Command::new("kubectl") .args([ "--context", @@ -4239,7 +4696,7 @@ fn read_backend_success_count(site: &str) -> Result Result Result Result> { + let metrics = read_workload_metrics(site, &format!("service/vcr-inference-{site}"), 8000)?; let mut found = false; let mut total = 0_u64; for line in metrics @@ -4293,6 +4755,43 @@ fn read_backend_success_count(site: &str) -> Result Result> { + let metrics = read_workload_metrics(site, "deployment/provider-gateway", 9901)?; + parse_provider_gateway_post_count(&metrics)? + .ok_or_else(|| format!("{site}: provider gateway did not expose a POST request counter").into()) +} + +/// Before the first request, Prometheus may not have a POST label series yet. +fn read_provider_gateway_post_count_cold(site: &str) -> Result> { + let metrics = read_workload_metrics(site, "deployment/provider-gateway", 9901)?; + Ok(parse_provider_gateway_post_count(&metrics)?.unwrap_or(0)) +} + +/// Sum provider-gateway POST ingress; `None` means no POST series has been created. +fn parse_provider_gateway_post_count(metrics: &str) -> Result, Box> { + if metrics.trim().is_empty() { + return Err("provider gateway returned an empty metrics response".into()); + } + let mut found = false; + let mut total = 0_u64; + for line in metrics + .lines() + .filter(|line| line.starts_with("praxis_http_requests_total{") && line.contains("method=\"POST\"")) + { + let value = line + .split_whitespace() + .last() + .ok_or("malformed provider gateway POST counter")? + .parse::()?; + total = total + .checked_add(value) + .ok_or("provider gateway POST counter overflow")?; + found = true; + } + Ok(found.then_some(total)) +} + /// Reserve and release an ephemeral loopback port for one metrics observation. fn find_local_tcp_port() -> Result> { let listener = TcpListener::bind("127.0.0.1:0")?; @@ -4342,6 +4841,8 @@ fn assert_provider_withdrawal_lifecycle( let start = Instant::now(); let mut facts = BTreeMap::new(); let mut original_selectors = Vec::new(); + let mut original_consumer_endpoint_mode = None; + let mut original_provider_hop_endpoints = None; let mut embedded_restore_prior_log_observations = 0; let body = (|| -> Result<(), Box> { for (site, resource) in PROVIDER_RESOURCES { @@ -4504,6 +5005,46 @@ fn assert_provider_withdrawal_lifecycle( } facts.insert("withdrawn_provider_b_c_selectors".to_owned(), serde_json::json!(true)); write_lifecycle_evidence(context, &facts)?; + + let previous_mode = set_generated_consumer_provider_a_transport_mode("plaintext")?; + original_consumer_endpoint_mode = Some(previous_mode.clone()); + if previous_mode != "mutual_tls" { + return Err(format!( + "negative consumer-config probe expected Provider A mutual_tls before injection, got {previous_mode:?}" + ) + .into()); + } + facts.insert( + "consumer_config_failure_injected".to_owned(), + serde_json::json!({ + "provider_cluster": "vcr-provider-a-provider", + "transport_mode": "plaintext", + "sni_remains_configured": true + }), + ); + write_lifecycle_evidence(context, &facts)?; + let consumer_config_failure = wait_for_generated_consumer_config_error("PlaintextWithSni")?; + facts.insert("consumer_config_failure_status".to_owned(), consumer_config_failure); + write_lifecycle_evidence(context, &facts)?; + + let invalid_hops = serde_json::json!([{ + "cluster": "vcr-provider-a-provider", + "transport": { + "mode": "plaintext", + "sni": "provider-a.grid.internal" + } + }]); + original_provider_hop_endpoints = Some(patch_embedded_provider_hop_endpoints(Some(&invalid_hops))?); + facts.insert( + "invalid_provider_hop_metadata_injected".to_owned(), + serde_json::json!({ + "cluster": "vcr-provider-a-provider", + "transport_mode": "plaintext", + "sni": "provider-a.grid.internal" + }), + ); + write_lifecycle_evidence(context, &facts)?; + let empty = wait_for_overlay_clusters(&BTreeSet::new(), Some(&fallback.semantic_revision))?; wait_for_consumer_gateway_revision(&empty.semantic_revision)?; facts.insert("empty_revision".to_owned(), serde_json::json!(empty.semantic_revision)); @@ -4511,6 +5052,10 @@ fn assert_provider_withdrawal_lifecycle( "empty_candidate_count".to_owned(), serde_json::json!(empty.candidates.len()), ); + facts.insert( + "empty_overlay_published_during_consumer_config_failure".to_owned(), + serde_json::json!(true), + ); facts.insert( "praxis_empty_serving_revision".to_owned(), serde_json::json!(empty.semantic_revision), @@ -4521,11 +5066,16 @@ fn assert_provider_withdrawal_lifecycle( embedded_serving_digest.as_deref(), Duration::from_secs(180), )?; - require_serving_provider_hops(&serving)?; + require_empty_serving_provider_hops(&serving)?; let log = wait_for_embedded_serving_log(&serving, &cluster_context(CONSUMER_SITE), 0, Duration::from_secs(90))?; + let warning = wait_for_empty_overlay_invalid_hop_warning(Duration::from_secs(45))?; facts.insert("embedded_empty_serving_config".to_owned(), serving); facts.insert("embedded_empty_serving_log".to_owned(), serde_json::json!(log)); + facts.insert( + "invalid_provider_hop_metadata_ignored".to_owned(), + serde_json::json!(warning), + ); embedded_serving_digest = Some( read_grid_serving_snapshot(&cluster_context(CONSUMER_SITE))? .get("digest") @@ -4534,6 +5084,46 @@ fn assert_provider_withdrawal_lifecycle( .to_owned(), ); } + if let Some(previous) = original_provider_hop_endpoints.take() { + let prior = if previous.is_null() { None } else { Some(&previous) }; + let injected = patch_embedded_provider_hop_endpoints(prior)?; + if injected + != serde_json::json!([{ + "cluster": "vcr-provider-a-provider", + "transport": { + "mode": "plaintext", + "sni": "provider-a.grid.internal" + } + }]) + { + return Err("embedded provider-hop declaration changed unexpectedly before restoration".into()); + } + facts.insert( + "provider_hop_metadata_restored_before_provider_restore".to_owned(), + serde_json::json!(true), + ); + } + write_lifecycle_evidence(context, &facts)?; + + let previous_mode = set_generated_consumer_provider_a_transport_mode( + original_consumer_endpoint_mode + .as_deref() + .ok_or("negative consumer-config probe lost its original transport mode")?, + )?; + if previous_mode != "plaintext" { + return Err( + format!("consumer-config probe expected plaintext before restore, got {previous_mode:?}").into(), + ); + } + original_consumer_endpoint_mode = None; + let restored_consumer_config = wait_for_generated_consumer_config()?; + facts.insert( + "consumer_config_restored_after_negative_probe".to_owned(), + serde_json::json!({ + "phase": restored_consumer_config.pointer("/status/phase"), + "observed_generation": restored_consumer_config.pointer("/status/observedGeneration") + }), + ); write_lifecycle_evidence(context, &facts)?; for (site, _) in PROVIDER_RESOURCES { @@ -4544,10 +5134,18 @@ fn assert_provider_withdrawal_lifecycle( .iter() .map(|(site, _)| Ok(((*site).to_owned(), read_backend_success_count(site)?))) .collect::>>()?; + let gateway_posts_before: BTreeMap = PROVIDER_RESOURCES + .iter() + .map(|(site, _)| Ok(((*site).to_owned(), read_provider_gateway_post_count(site)?))) + .collect::>>()?; facts.insert( "backend_success_total_before_no_route".to_owned(), serde_json::json!(metrics_before), ); + facts.insert( + "provider_gateway_posts_before_no_route".to_owned(), + serde_json::json!(gateway_posts_before), + ); write_lifecycle_evidence(context, &facts)?; let mut no_route_results = Vec::new(); @@ -4598,6 +5196,16 @@ fn assert_provider_withdrawal_lifecycle( .iter() .map(|(site, _)| Ok(((*site).to_owned(), read_backend_success_count(site)?))) .collect::>>()?; + let gateway_posts_after: BTreeMap = PROVIDER_RESOURCES + .iter() + .map(|(site, _)| Ok(((*site).to_owned(), read_provider_gateway_post_count(site)?))) + .collect::>>()?; + if gateway_posts_before != gateway_posts_after { + return Err(format!( + "provider gateway POST counters changed while no route was serving: before={gateway_posts_before:?}, after={gateway_posts_after:?}" + ) + .into()); + } if metrics_before != metrics_after { return Err(format!("backend request counters changed while no route was serving: before={metrics_before:?}, after={metrics_after:?}").into()); } @@ -4605,6 +5213,10 @@ fn assert_provider_withdrawal_lifecycle( "backend_success_total_after_no_route".to_owned(), serde_json::json!(metrics_after), ); + facts.insert( + "provider_gateway_posts_after_no_route".to_owned(), + serde_json::json!(gateway_posts_after), + ); facts.insert("backend_non_contact_proven".to_owned(), serde_json::json!(true)); if embedded_gateway_enabled { facts.insert( @@ -4665,12 +5277,26 @@ fn assert_provider_withdrawal_lifecycle( Ok(()) })(); - let mut restoration_error = None; + let mut consumer_config_restoration_error = None; + if let Some(mode) = original_consumer_endpoint_mode.take() { + if let Err(error) = set_generated_consumer_provider_a_transport_mode(&mode) + .and_then(|_| wait_for_generated_consumer_config().map(|_| ())) + { + consumer_config_restoration_error = Some(error.to_string()); + } + } + let mut restoration_error = consumer_config_restoration_error.clone(); + if let Some(previous) = original_provider_hop_endpoints.take() { + let prior = if previous.is_null() { None } else { Some(&previous) }; + if let Err(error) = patch_embedded_provider_hop_endpoints(prior) { + restoration_error.get_or_insert_with(|| format!("provider-hop declaration: {error}")); + } + } if !original_selectors.is_empty() { let before_restore = read_cluster_overlay("provider-a").ok(); for state in &original_selectors { if let Err(error) = patch_provider_selector(state, &state.selector_value) { - restoration_error = Some(format!("{}: {error}", state.site)); + restoration_error.get_or_insert_with(|| format!("{}: {error}", state.site)); } } if restoration_error.is_none() { @@ -4699,6 +5325,10 @@ fn assert_provider_withdrawal_lifecycle( "selector_restoration_error".to_owned(), serde_json::json!(restoration_error), ); + facts.insert( + "consumer_config_restoration_error".to_owned(), + serde_json::json!(consumer_config_restoration_error), + ); facts.insert( "lifecycle_error".to_owned(), serde_json::json!(body.as_ref().err().map(ToString::to_string)), @@ -5147,6 +5777,38 @@ fn collect_embedded_gateway_image_evidence() -> Result, Box Result> { + let mut docker = Command::new("docker") + .args(["image", "save", "--platform", "linux/amd64", image]) + .stdout(Stdio::piped()) + .spawn()?; + let archive = docker.stdout.take().ok_or("docker image save did not provide stdout")?; + let manifest = Command::new("tar") + .args(["-xOf", "-", "manifest.json"]) + .stdin(Stdio::from(archive)) + .output()?; + let docker_status = docker.wait()?; + if !manifest.status.success() || !docker_status.success() { + return Err(format!("could not inspect the local linux/amd64 image config for {image}").into()); + } + image_config_digest_from_manifest(&manifest.stdout) +} + +/// Docker's OCI archive names the config blob separately from its index digest. +fn image_config_digest_from_manifest(manifest: &[u8]) -> Result> { + let entries: serde_json::Value = serde_json::from_slice(manifest)?; + let config = entries + .as_array() + .and_then(|entries| entries.first()) + .and_then(|entry| entry.get("Config")) + .and_then(serde_json::Value::as_str) + .and_then(|path| path.strip_prefix("blobs/sha256/")) + .filter(|digest| digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit())) + .ok_or("image archive has no valid OCI config digest")?; + Ok(format!("sha256:{config}")) +} + /// Check whether a deployment exists without treating a missing optional workload as an error. fn deployment_exists(context: &str, deployment: &str) -> Result> { let output = Command::new("kubectl") @@ -5177,7 +5839,7 @@ fn deployment_exists(context: &str, deployment: &str) -> Result Vec<(&'static str, String)> { +pub(super) fn image_evidence_deployments(cluster: &str) -> Vec<(&'static str, String)> { let mut deployments = vec![ ("operator", "grid-operator".to_owned()), ("provider_gateway", "provider-gateway".to_owned()), @@ -5194,7 +5856,10 @@ fn image_evidence_deployments(cluster: &str) -> Vec<(&'static str, String)> { clippy::too_many_lines, reason = "deployment identity, requested images, and every ready runtime image ID are captured together" )] -fn deployment_runtime_image_evidence(context: &str, deployment: &str) -> Result> { +pub(super) fn deployment_runtime_image_evidence( + context: &str, + deployment: &str, +) -> Result> { let deployment_output = Command::new("kubectl") .args([ "get", @@ -5227,6 +5892,13 @@ fn deployment_runtime_image_evidence(context: &str, deployment: &str) -> Result< .iter() .filter_map(|container| Some((container.get("name")?.as_str()?, container.get("image")?.as_str()?))) .collect(); + let verify_local = std::env::var("GRID_XTASK_IMAGE_PULL_POLICY").unwrap_or_else(|_| "Never".to_owned()) == "Never"; + let mut expected_by_image = BTreeMap::new(); + if verify_local { + for image in requested_by_name.values() { + expected_by_image.insert(*image, local_image_config_digest(image)?); + } + } let selector_arg = selector .iter() .filter_map(|(key, value)| Some(format!("{key}={}", value.as_str()?))) @@ -5295,10 +5967,23 @@ fn deployment_runtime_image_evidence(context: &str, deployment: &str) -> Result< .and_then(serde_json::Value::as_str) .filter(|image_id| !image_id.is_empty()) .ok_or_else(|| format!("pod {pod_name} container {name} has no runtime imageID"))?; + let requested = requested_by_name + .get(name) + .ok_or_else(|| format!("pod {pod_name} container {name} is absent from the Deployment template"))?; + let expected = expected_by_image.get(requested); + if let Some(expected) = expected + && image_id != expected + { + return Err(format!( + "pod {pod_name} container {name} runs {image_id}, but local image {requested} has config digest {expected}" + )); + } Ok(serde_json::json!({ "name": name, - "requested": requested_by_name.get(name), + "requested": requested, "imageID": image_id, + "expectedLocalConfigDigest": expected, + "sourceMatched": expected.is_some(), })) }) .collect::>()?; @@ -5330,6 +6015,33 @@ mod tests { } } + #[test] + fn provider_gateway_post_counter_counts_failed_and_successful_statuses() { + let metrics = concat!( + "praxis_http_requests_total{method=\"GET\",status_class=\"2xx\"} 8\n", + "praxis_http_requests_total{method=\"POST\",status_class=\"2xx\"} 3\n", + "praxis_http_requests_total{method=\"POST\",status_class=\"5xx\"} 2\n", + ); + assert_eq!(parse_provider_gateway_post_count(metrics).unwrap(), Some(5)); + assert_eq!( + parse_provider_gateway_post_count("praxis_http_requests_total{method=\"GET\"} 8").unwrap(), + None, + "a cold gateway has no POST series until it handles a POST" + ); + assert!(parse_provider_gateway_post_count("").is_err()); + } + + #[test] + fn image_manifest_config_digest_is_distinct_from_index_digest() { + let config = "a".repeat(64); + let manifest = format!(r#"[{{"Config":"blobs/sha256/{config}"}}]"#); + assert_eq!( + image_config_digest_from_manifest(manifest.as_bytes()).unwrap(), + format!("sha256:{config}") + ); + assert!(image_config_digest_from_manifest(br#"[{"Config":"../escape"}]"#).is_err()); + } + #[test] fn embedded_consumer_charts_resolve_from_grid_repository_root() { let root = grid_repository_root().unwrap_or_else(|_| std::process::abort()); @@ -5575,6 +6287,10 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "keeps the declaration and disabled consumer config assertions together" + )] fn embedded_gateway_declares_mtls_provider_hops_separately_from_disabled_consumer_config() { let endpoints = PROVIDER_RESOURCES .iter() @@ -5629,6 +6345,62 @@ mod tests { assert!(require_serving_provider_hops(&extra).is_err()); } + #[test] + fn empty_embedded_serving_snapshot_omits_provider_hop_trust() { + let omitted = serde_json::json!({"serving_config": {"candidates": []}}); + assert!(require_empty_serving_provider_hops(&omitted).is_ok()); + + let explicit_empty = serde_json::json!({ + "serving_config": { + "candidates": [], + "provider_hop_clusters": [], + "provider_hop_sni": {} + } + }); + assert!(require_empty_serving_provider_hops(&explicit_empty).is_ok()); + + let stale_trust = serde_json::json!({ + "serving_config": { + "candidates": [], + "provider_hop_clusters": ["provider-a"], + "provider_hop_sni": {"provider-a": "provider-a.example"} + } + }); + assert!(require_empty_serving_provider_hops(&stale_trust).is_err()); + + let malformed = serde_json::json!({ + "serving_config": {"candidates": [], "provider_hop_clusters": {}} + }); + assert!(require_empty_serving_provider_hops(&malformed).is_err()); + } + + #[test] + fn curl_probe_waits_for_a_terminal_container_status() { + let running = serde_json::json!({"status": {"phase": "Running"}}); + assert_eq!(probe_pod_exit_code(&running), None); + + let succeeded = serde_json::json!({"status": {"phase": "Succeeded"}}); + assert_eq!(probe_pod_exit_code(&succeeded), Some(0)); + + let failed = serde_json::json!({ + "status": { + "phase": "Failed", + "containerStatuses": [{"state": {"terminated": {"exitCode": 22}}}] + } + }); + assert_eq!(probe_pod_exit_code(&failed), Some(22)); + } + + #[test] + fn response_diagnostics_retain_status_and_header_names_not_values() { + let response = b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nX-Grid-Provider-Gateway: provider-a\r\n\r\n{\"private\":\"body\"}\nGRID258_HTTP_STATUS:200\n"; + assert_eq!(response_status_line(response).as_deref(), Some("HTTP/1.1 200 OK")); + assert_eq!( + response_header_names(response), + vec!["content-type".to_owned(), "x-grid-provider-gateway".to_owned()] + ); + } + #[test] fn restored_provider_attribution_uses_site_labels_not_cluster_names() { let sites = expected_provider_attributions(); diff --git a/xtask/src/env/static_weighted_qualification.rs b/xtask/src/env/static_weighted_qualification.rs index 7a68facf9..e97b5d556 100644 --- a/xtask/src/env/static_weighted_qualification.rs +++ b/xtask/src/env/static_weighted_qualification.rs @@ -13,7 +13,9 @@ use std::{ use serde::Serialize; -use super::{DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, safe_truncate_str}; +use super::{ + DemoMode, GlbDemoOptions, certs, glb, kubectl, operator, provider_traffic_qualification, safe_truncate_str, +}; // ----------------------------------------------------------------------------- // Constants @@ -3738,94 +3740,16 @@ pub(crate) fn run(forge_config: &Path, options: &GlbDemoOptions) -> Result<(), B } } -/// Collect actual image evidence from the deployed clusters. -#[expect( - clippy::too_many_lines, - reason = "Evidence collection queries the bounded set of deployed component images." -)] +/// Collect requested references and runtime image IDs from every required pod. fn collect_image_evidence() -> Result, Box> { let mut image_evidence = BTreeMap::new(); - for cluster in CLUSTERS { let context = cluster_context(cluster); - - // Get grid operator image - let operator_output = Command::new("kubectl") - .args([ - "get", - "deployment/grid-operator", - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; - - if operator_output.status.success() { - let operator_image = String::from_utf8_lossy(&operator_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_operator"), operator_image); - } - - // Get consumer gateway image - let consumer_output = Command::new("kubectl") - .args([ - "get", - "deployment/consumer-gateway", - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; - - if consumer_output.status.success() { - let consumer_image = String::from_utf8_lossy(&consumer_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_consumer_gateway"), consumer_image); - } - - // Get provider gateway image - let provider_output = Command::new("kubectl") - .args([ - "get", - "deployment/provider-gateway", - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; - - if provider_output.status.success() { - let provider_image = String::from_utf8_lossy(&provider_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_provider_gateway"), provider_image); - } - - // Get VCR inference image - let vcr_output = Command::new("kubectl") - .args([ - "get", - &format!("deployment/vcr-inference-{cluster}"), - "--context", - &context, - "-n", - "grid-system", - "-o", - "jsonpath={.spec.template.spec.containers[0].image}", - ]) - .output()?; - - if vcr_output.status.success() { - let mock_image = String::from_utf8_lossy(&vcr_output.stdout).trim().to_owned(); - image_evidence.insert(format!("{cluster}_vcr_inference"), mock_image); + for (component, deployment) in provider_traffic_qualification::image_evidence_deployments(cluster) { + let evidence = provider_traffic_qualification::deployment_runtime_image_evidence(&context, &deployment)?; + image_evidence.insert(format!("{cluster}_{component}"), evidence); } } - Ok(image_evidence) } From b0384384e2f4cdb3369461eff8f521bafc08ab87 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 3 Oct 2026 01:31:04 +0000 Subject: [PATCH 04/10] fix(gateway): reject ambiguous provider-hop backend trust Signed-off-by: Brent Salisbury --- gateway/src/main.rs | 48 +++++++++---- .../src/env/provider_traffic_qualification.rs | 69 +++++++++++-------- 2 files changed, 77 insertions(+), 40 deletions(-) diff --git a/gateway/src/main.rs b/gateway/src/main.rs index c06def60b..648d11fe1 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -270,6 +270,7 @@ struct BackendClientCert { #[expect(clippy::too_many_lines, reason = "keeps the backend transport checks together")] fn provider_hop_backends(config: &Config) -> Result, praxis_filter::FilterError> { let mut backends = BTreeMap::new(); + let mut unverified = BTreeSet::new(); for chain in &config.filter_chains { if !chain.filters.iter().any(|filter| filter.filter_type == "grid_site_route") { continue; @@ -280,18 +281,24 @@ fn provider_hop_backends(config: &Config) -> Result, pr format!("grid: parsing load_balancer backends: {error}").into() })?; for backend in parsed.clusters { - let Some(tls) = backend.tls else { continue }; - if !tls.verify - || tls.sni.trim().is_empty() - || tls.ca.as_ref().is_none_or(|ca| ca.ca_path.trim().is_empty()) - || tls.client_cert.as_ref().is_none_or(|cert| { - cert.cert_path.trim().is_empty() || cert.key_path.trim().is_empty() - }) - { - continue; - } - if backends.insert(backend.name.clone(), tls.sni).is_some() { - return Err(format!("grid: duplicate verified backend {:?}", backend.name).into()); + let verified_sni = backend.tls.filter(|tls| { + tls.verify + && !tls.sni.trim().is_empty() + && tls.ca.as_ref().is_some_and(|ca| !ca.ca_path.trim().is_empty()) + && tls + .client_cert + .as_ref() + .is_some_and(|cert| !cert.cert_path.trim().is_empty() && !cert.key_path.trim().is_empty()) + }); + if let Some(tls) = verified_sni { + if unverified.contains(&backend.name) || backends.insert(backend.name.clone(), tls.sni).is_some() { + return Err(format!("grid: ambiguous provider-hop backend {:?}", backend.name).into()); + } + } else { + if backends.contains_key(&backend.name) { + return Err(format!("grid: ambiguous provider-hop backend {:?}", backend.name).into()); + } + unverified.insert(backend.name); } } } @@ -331,6 +338,23 @@ mod tests { use super::{USAGE, config_arg, otlp_headers_present, validate_otlp_endpoint_transport_values}; + #[test] + fn provider_hop_backends_reject_verified_plaintext_name_collision() { + let verified = " tls:\n sni: provider-a.grid.internal\n verify: true\n ca: { ca_path: /tls/ca.crt }\n client_cert: { cert_path: /tls/tls.crt, key_path: /tls/tls.key }\n"; + let plaintext = " endpoints: [provider-a:80]\n"; + for (first, second) in [(verified, plaintext), (plaintext, verified)] { + let mut config = config_with_backend(first); + let mut other = config_with_backend(second).filter_chains.remove(0); + other.name = "other-grid".to_owned(); + config.filter_chains.push(other); + assert!( + provider_hop_backends(&config) + .err() + .is_some_and(|error| error.to_string().contains("ambiguous provider-hop backend")) + ); + } + } + fn parse(args: &[&str]) -> Result, String> { config_arg(args.iter().map(|arg| (*arg).to_owned())) } diff --git a/xtask/src/env/provider_traffic_qualification.rs b/xtask/src/env/provider_traffic_qualification.rs index 1f238303c..460397e64 100644 --- a/xtask/src/env/provider_traffic_qualification.rs +++ b/xtask/src/env/provider_traffic_qualification.rs @@ -485,25 +485,33 @@ fn curl_pod_overrides(pod_name: &str, curl_args: &[&str]) -> String { /// `kubectl run`'s attached stream can lose output from a short-lived curl /// container. Read logs only after Kubernetes reports a terminated container. struct ProbeExitStatus { + /// Terminated curl process exit code. code: i32, } impl ProbeExitStatus { + /// Whether curl exited successfully. fn success(&self) -> bool { self.code == 0 } } +/// Completed curl probe output collected from its temporary Pod. struct CurlProbeOutput { + /// Curl process status. status: ProbeExitStatus, + /// Captured standard output. stdout: Vec, + /// Captured standard error. stderr: Vec, } /// Delete exactly the temporary probe Pod created by `run_curl_probe`. -struct ProbePodCleanup<'a> { - context: &'a str, - name: &'a str, +struct ProbePodCleanup<'probe> { + /// Kubernetes context containing the temporary Pod. + context: &'probe str, + /// Name of the temporary Pod. + name: &'probe str, } impl Drop for ProbePodCleanup<'_> { @@ -546,6 +554,7 @@ fn probe_pod_exit_code(pod: &serde_json::Value) -> Option { } /// Run a restricted curl Pod, collect its completed logs, and then remove it. +#[expect(clippy::too_many_lines, reason = "probe lifecycle and cleanup are one operation")] fn run_curl_probe( context: &str, pod_name: &str, @@ -2829,6 +2838,7 @@ fn enable_generated_consumer_config() -> Result<(), Box> } /// Change only the run-owned Provider A consumer endpoint transport and return its previous mode. +#[expect(clippy::too_many_lines, reason = "reads and patches the run-owned consumer endpoint")] fn set_generated_consumer_provider_a_transport_mode(mode: &str) -> Result> { let context = cluster_context("provider-a"); let output = Command::new("kubectl") @@ -2876,9 +2886,9 @@ fn set_generated_consumer_provider_a_transport_mode(mode: &str) -> Result Result Result> { @@ -3310,6 +3321,7 @@ fn patch_embedded_provider_hop_endpoints( /// Wait for the controller's warning that invalid optional hop metadata was /// ignored while publishing an empty serving revision. +#[expect(clippy::too_many_lines, reason = "polls controller logs for the scoped warning")] fn wait_for_empty_overlay_invalid_hop_warning(timeout: Duration) -> Result> { let context = cluster_context(CONSUMER_SITE); let deadline = Instant::now() + timeout; @@ -4303,7 +4315,6 @@ fn run_provider_traffic_scenarios(mode: DemoMode, context: &ProviderTrafficConte /// the request itself is the only source of the attribution counts. #[expect( clippy::too_many_lines, - clippy::unnecessary_wraps, reason = "The assertion framework requires a fallible, named traffic proof boundary." )] fn assert_provider_gateway_round_robin() -> AssertionResult { @@ -5105,14 +5116,14 @@ fn assert_provider_withdrawal_lifecycle( } write_lifecycle_evidence(context, &facts)?; - let previous_mode = set_generated_consumer_provider_a_transport_mode( + let restored_mode = set_generated_consumer_provider_a_transport_mode( original_consumer_endpoint_mode .as_deref() .ok_or("negative consumer-config probe lost its original transport mode")?, )?; - if previous_mode != "plaintext" { + if restored_mode != "plaintext" { return Err( - format!("consumer-config probe expected plaintext before restore, got {previous_mode:?}").into(), + format!("consumer-config probe expected plaintext before restore, got {restored_mode:?}").into(), ); } original_consumer_endpoint_mode = None; @@ -5278,12 +5289,11 @@ fn assert_provider_withdrawal_lifecycle( })(); let mut consumer_config_restoration_error = None; - if let Some(mode) = original_consumer_endpoint_mode.take() { - if let Err(error) = set_generated_consumer_provider_a_transport_mode(&mode) + if let Some(mode) = original_consumer_endpoint_mode.take() + && let Err(error) = set_generated_consumer_provider_a_transport_mode(&mode) .and_then(|_| wait_for_generated_consumer_config().map(|_| ())) - { - consumer_config_restoration_error = Some(error.to_string()); - } + { + consumer_config_restoration_error = Some(error.to_string()); } let mut restoration_error = consumer_config_restoration_error.clone(); if let Some(previous) = original_provider_hop_endpoints.take() { @@ -5967,20 +5977,20 @@ pub(super) fn deployment_runtime_image_evidence( .and_then(serde_json::Value::as_str) .filter(|image_id| !image_id.is_empty()) .ok_or_else(|| format!("pod {pod_name} container {name} has no runtime imageID"))?; - let requested = requested_by_name + let requested_image = requested_by_name .get(name) .ok_or_else(|| format!("pod {pod_name} container {name} is absent from the Deployment template"))?; - let expected = expected_by_image.get(requested); + let expected = expected_by_image.get(requested_image); if let Some(expected) = expected && image_id != expected { return Err(format!( - "pod {pod_name} container {name} runs {image_id}, but local image {requested} has config digest {expected}" + "pod {pod_name} container {name} runs {image_id}, but local image {requested_image} has config digest {expected}" )); } Ok(serde_json::json!({ "name": name, - "requested": requested, + "requested": requested_image, "imageID": image_id, "expectedLocalConfigDigest": expected, "sourceMatched": expected.is_some(), @@ -6022,13 +6032,13 @@ mod tests { "praxis_http_requests_total{method=\"POST\",status_class=\"2xx\"} 3\n", "praxis_http_requests_total{method=\"POST\",status_class=\"5xx\"} 2\n", ); - assert_eq!(parse_provider_gateway_post_count(metrics).unwrap(), Some(5)); + assert_eq!(parse_provider_gateway_post_count(metrics).ok(), Some(Some(5))); assert_eq!( - parse_provider_gateway_post_count("praxis_http_requests_total{method=\"GET\"} 8").unwrap(), - None, + parse_provider_gateway_post_count("praxis_http_requests_total{method=\"GET\"} 8").ok(), + Some(None), "a cold gateway has no POST series until it handles a POST" ); - assert!(parse_provider_gateway_post_count("").is_err()); + assert_eq!(parse_provider_gateway_post_count("").ok(), None); } #[test] @@ -6036,10 +6046,13 @@ mod tests { let config = "a".repeat(64); let manifest = format!(r#"[{{"Config":"blobs/sha256/{config}"}}]"#); assert_eq!( - image_config_digest_from_manifest(manifest.as_bytes()).unwrap(), - format!("sha256:{config}") + image_config_digest_from_manifest(manifest.as_bytes()).ok(), + Some(format!("sha256:{config}")) + ); + assert_eq!( + image_config_digest_from_manifest(br#"[{"Config":"../escape"}]"#).ok(), + None ); - assert!(image_config_digest_from_manifest(br#"[{"Config":"../escape"}]"#).is_err()); } #[test] @@ -6348,7 +6361,7 @@ mod tests { #[test] fn empty_embedded_serving_snapshot_omits_provider_hop_trust() { let omitted = serde_json::json!({"serving_config": {"candidates": []}}); - assert!(require_empty_serving_provider_hops(&omitted).is_ok()); + assert!(matches!(require_empty_serving_provider_hops(&omitted), Ok(()))); let explicit_empty = serde_json::json!({ "serving_config": { @@ -6357,7 +6370,7 @@ mod tests { "provider_hop_sni": {} } }); - assert!(require_empty_serving_provider_hops(&explicit_empty).is_ok()); + assert!(matches!(require_empty_serving_provider_hops(&explicit_empty), Ok(()))); let stale_trust = serde_json::json!({ "serving_config": { From 8675ba97f2da427c1ced3184d35472f05637620f Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sat, 3 Oct 2026 05:06:11 +0000 Subject: [PATCH 05/10] fix(routing): publish authoritative empty overlays by default Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 9 ----- charts/praxis-gateway/README.md | 10 +++--- deploy/crds/gridnetwork.yaml | 9 ----- docs/architecture/consumer-config.md | 12 +++---- docs/architecture/crds.md | 14 ++++---- operator/src/controller/grid_network.rs | 1 - operator/src/crd/grid_network.rs | 34 ------------------- .../grid-provider-traffic/README.md | 10 +++--- .../grid-provider-traffic/forge.yaml | 3 -- .../src/env/provider_traffic_qualification.rs | 1 - 10 files changed, 21 insertions(+), 82 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index 9bcdacf85..72c963b5b 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -492,15 +492,6 @@ spec: - transport type: object type: array - supportsEmptyOverlay: - default: false - description: |- - Explicitly attest that every consumer of this gateway's routing overlay - accepts a valid versioned snapshot with `candidates: []` as a no-route - decision. Defaults to `false` for compatibility with released Praxis - images that reject empty candidate lists. The operator retains the last - distributed revision until this is enabled. - type: boolean required: - name - namespace diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index 753b66d04..5d05c08b1 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -397,11 +397,11 @@ balancing and includes Basic Auth. It does not include the optional qualification. That qualification is not supported by this default image; AGN does not publish a replacement AI rollup. -The default image also predates empty versioned routing snapshots. Keep -`GridNetwork.spec.gatewayRefs[].supportsEmptyOverlay` unset/false when using this -default; Grid will retain the last distributed overlay rather than sending a -snapshot this image rejects. Upgrade and roll every consumer to an image with -empty-snapshot support before explicitly enabling that GridNetwork capability. +The default image predates empty versioned routing snapshots and is not +compatible with Grid's authoritative no-route publication. The paired Praxis AI +change must be released, and this chart's default image must be updated to that +compatible release, before the next Grid release. Upgrade and roll every +consumer of a Grid-managed overlay before deploying that Grid version. Generated consumer credentials use a separate two-step opt-in: set `consumerConfig.enableProjectedCredentials: true`, let Grid render the filter, diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 6f40ecf6f..4eda4c386 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -484,15 +484,6 @@ spec: - transport type: object type: array - supportsEmptyOverlay: - default: false - description: |- - Explicitly attest that every consumer of this gateway's routing overlay - accepts a valid versioned snapshot with `candidates: []` as a no-route - decision. Defaults to `false` for compatibility with released Praxis - images that reject empty candidate lists. The operator retains the last - distributed revision until this is enabled. - type: boolean required: - name - namespace diff --git a/docs/architecture/consumer-config.md b/docs/architecture/consumer-config.md index 2511b82de..1061c43a7 100644 --- a/docs/architecture/consumer-config.md +++ b/docs/architecture/consumer-config.md @@ -196,15 +196,15 @@ an empty list in arbitrary startup YAML: | Consumer path | Route-state source | Valid no-route state | Runtime update behavior | |---|---|---|---| -| Praxis `intelligent_route` with `overlay_file` | Grid's versioned `routing-overlay.json` ConfigMap | A valid versioned envelope whose `overlay.candidates` is `[]` | Praxis validates and atomically serves the new snapshot; malformed replacements retain last-known-good. Grid distributes an empty revision only after `GatewayRef.supportsEmptyOverlay: true`. | +| Praxis `intelligent_route` with `overlay_file` | Grid's versioned `routing-overlay.json` ConfigMap | A valid versioned envelope whose `overlay.candidates` is `[]` | Praxis validates and atomically serves the new snapshot; malformed replacements retain last-known-good. | | Generated `GatewayRef.consumerConfig` | The same scoped versioned overlay; generated `praxis.yaml` supplies filter and endpoint plumbing | The same empty envelope | Candidate-only updates hot reload. Credential-bearing revisions are held until `enableProjectedCredentials` has been rolled out and `supportsProjectedCredentials: true` attests the filter and Secret mounts are active. Listener, endpoint/TLS, and filter-pipeline changes still require the consumer owner to reload or roll out its Praxis configuration. | | Embedded `grid-gateway` `grid_site_route` filter | The operator-published `grid-serving--` ConfigMap | A valid serving config with `candidates: []` | The running gateway watches the projected serving file and atomically replaces its candidate snapshot and provider-hop allowlist. Malformed updates retain the previous snapshot. Provider-hop trust is declared separately with `GatewayRef.providerHopEndpoints`. | -`GatewayRef.supportsEmptyOverlay` defaults to `false`. Set it to `true` only -after every data-plane consumer of that gateway's overlay has been upgraded and -rolled out with empty-snapshot support. Until then, Grid retains the last -distributed revision and reports `EmptyOverlayUnsupported`; this is an explicit -compatibility boundary, not a successful withdrawal. +Grid publishes authoritative empty revisions without a capability flag. Every +consumer of a gateway's overlay must therefore run an image with empty-snapshot +support before this Grid version is deployed. The chart-default Praxis AI 0.4.0 +image is not compatible; the paired AI change and a compatible image release +are prerequisites for the next Grid release. The embedded gateway's filter chain and upstream cluster definitions remain in its startup Praxis configuration, but its changing candidate list is not a diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index f7fb39eba..b9932f22b 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -27,7 +27,6 @@ spec: - name: inference-gw namespace: praxis-system localSiteName: cluster-east # optional; defaults to network name - supportsEmptyOverlay: true # set only after every consumer accepts empty v1 snapshots consumerConfig: # optional; opt-in consumer Praxis config generation enabled: true enableProjectedCredentials: true # generate filter, then roll out consumer @@ -315,13 +314,12 @@ Praxis `ConfigMap` generation. | `tlsCertMountPath` | `/etc/praxis/tls` | Base path for mounted TLS files used when a `clusterEndpoints[]` entry uses `mutual_tls` transport. | | `listenerPort` | `8080` | HTTP port for the generated `listeners[0].address` (`0.0.0.0:{listenerPort}`). | -`GatewayRef.supportsEmptyOverlay` is a separate, default-false capability gate. -Released Praxis images that reject an empty candidate list must not receive an -empty routing revision. Upgrade and roll every consumer to an image that accepts -the versioned empty snapshot first; only then set `supportsEmptyOverlay: true`. -Until opted in, Grid retains the last distributed revision and reports -`EmptyOverlayUnsupported` in `status.overlayStatus[]` rather than claiming the -withdrawal was served. Non-empty overlay updates are unaffected. +Grid publishes a valid empty versioned overlay when no candidates remain. +Every consumer of that overlay must use a Praxis AI image that accepts empty +versioned snapshots; older images, including the chart-default 0.4.0 image, +are incompatible. Upgrade and roll out the consumers before deploying this +Grid version. Malformed or undeliverable updates still retain the last valid +revision; an authoritative empty candidate set does not. When `enabled: true`, the `GridNetwork` controller renders a `praxis.yaml`-keyed `ConfigMap` in the gateway namespace on each reconcile. The generated config is a diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 14aef76a8..6dfbdeee6 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -8811,7 +8811,6 @@ mod tests { name: name.to_owned(), namespace: ns.to_owned(), local_site_name: None, - supports_empty_overlay: false, provider_hop_endpoints: Vec::new(), consumer_config: None, } diff --git a/operator/src/crd/grid_network.rs b/operator/src/crd/grid_network.rs index 74d627565..fb2ffd374 100644 --- a/operator/src/crd/grid_network.rs +++ b/operator/src/crd/grid_network.rs @@ -746,14 +746,6 @@ pub struct GatewayRef { #[serde(default)] pub local_site_name: Option, - /// Explicitly attest that every consumer of this gateway's routing overlay - /// accepts a valid versioned snapshot with `candidates: []` as a no-route - /// decision. Defaults to `false` for compatibility with released Praxis - /// images that reject empty candidate lists. The operator retains the last - /// distributed revision until this is enabled. - #[serde(default)] - pub supports_empty_overlay: bool, - /// Explicit mTLS provider-hop endpoints for the embedded `grid-gateway` /// serving config. This allowlist is independent of `consumerConfig`, so a /// missing or disabled generated Praxis config cannot alter embedded @@ -1788,31 +1780,6 @@ mod tests { gateway_ref_properties.contains_key("localSiteName"), "CRD schema must include localSiteName field on GatewayRef" ); - assert!( - gateway_ref_properties.contains_key("supportsEmptyOverlay"), - "CRD schema must include the empty-overlay capability gate" - ); - } - - #[test] - fn gateway_ref_empty_overlay_capability_defaults_closed_and_round_trips() { - let mut gw: GatewayRef = serde_json::from_value(serde_json::json!({ - "name": "gw", - "namespace": "ns" - })) - .unwrap_or_else(|_| std::process::abort()); - assert!( - !gw.supports_empty_overlay, - "old GatewayRef documents default to unsupported" - ); - gw.supports_empty_overlay = true; - let serialized = serde_json::to_value(gw).unwrap_or_else(|_| std::process::abort()); - assert_eq!( - serialized - .get("supportsEmptyOverlay") - .and_then(serde_json::Value::as_bool), - Some(true) - ); } // ----------------------------------------------------------------------- @@ -2091,7 +2058,6 @@ mod tests { name: "gw".to_owned(), namespace: "ns".to_owned(), local_site_name: None, - supports_empty_overlay: false, provider_hop_endpoints: Vec::new(), consumer_config: None, }; diff --git a/tests/e2e/topologies/grid-provider-traffic/README.md b/tests/e2e/topologies/grid-provider-traffic/README.md index c1c35692f..61674c2cc 100644 --- a/tests/e2e/topologies/grid-provider-traffic/README.md +++ b/tests/e2e/topologies/grid-provider-traffic/README.md @@ -173,12 +173,10 @@ provider-withdrawal lifecycle (fallback, a valid empty/no-route revision, backend non-contact, and restoration). `--quick` runs only the original six scenarios. -The run-owned `GatewayRef`s explicitly set `supportsEmptyOverlay: true` because -the qualification builds both consumers from the compatible source under test. -Production deployments must first upgrade and roll out every consumer; the -field defaults to false so an older released image (including the chart's -default AI 0.4.0) retains its prior revision instead of receiving an invalid -empty snapshot. +The qualification builds both consumers from compatible source under test. +Deployments must upgrade and roll out every consumer before using this Grid +version, because older images (including the chart-default AI 0.4.0) reject +the authoritative empty snapshot. Full mode also exercises the generated consumer credential lifecycle in two phases: it first renders `credential_inject` with an empty table and rolls out diff --git a/tests/e2e/topologies/grid-provider-traffic/forge.yaml b/tests/e2e/topologies/grid-provider-traffic/forge.yaml index 3ce837b7b..6af55785a 100644 --- a/tests/e2e/topologies/grid-provider-traffic/forge.yaml +++ b/tests/e2e/topologies/grid-provider-traffic/forge.yaml @@ -230,7 +230,6 @@ spec: - name: consumer-gateway namespace: grid-system localSiteName: provider-a - supportsEmptyOverlay: true gridSite: name: provider-a region: provider-a @@ -294,7 +293,6 @@ spec: - name: consumer-gateway namespace: grid-system localSiteName: provider-b - supportsEmptyOverlay: true gridSite: name: provider-b region: provider-b @@ -358,7 +356,6 @@ spec: - name: consumer-gateway namespace: grid-system localSiteName: provider-c - supportsEmptyOverlay: true gridSite: name: provider-c region: provider-c diff --git a/xtask/src/env/provider_traffic_qualification.rs b/xtask/src/env/provider_traffic_qualification.rs index 460397e64..4792942d9 100644 --- a/xtask/src/env/provider_traffic_qualification.rs +++ b/xtask/src/env/provider_traffic_qualification.rs @@ -3371,7 +3371,6 @@ fn embedded_serving_gateway_ref(endpoints: &[serde_json::Value]) -> serde_json:: "name": GRID_SERVING_GATEWAY, "namespace": GRID_SYSTEM_NS, "localSiteName": CONSUMER_SITE, - "supportsEmptyOverlay": true, "providerHopEndpoints": provider_hop_endpoints, "consumerConfig": { "enabled": false, From b621f18d7b79ed89e434fa95f63eb756518fdb7b Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sun, 4 Oct 2026 16:40:57 +0000 Subject: [PATCH 06/10] fix(gateway): integrate no-route serving with config reload Signed-off-by: Brent Salisbury --- gateway/ai-grid-filters/src/control.rs | 14 ++++++++++ gateway/ai-grid-filters/src/descriptor.rs | 1 + gateway/ai-grid-filters/src/serving.rs | 1 + gateway/src/main.rs | 31 ++++++++++++++++++++--- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index c7fe132df..da30c501a 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -228,6 +228,7 @@ pub(crate) struct Control { impl Control { /// Build the control plane for `config` without starting any poller. + #[cfg(test)] pub(crate) fn new(config: &GridServingConfig, start: StartPeer) -> Result { Self::new_with_backend_tls(config, start, BTreeMap::new()) } @@ -810,6 +811,7 @@ mod tests { kind: CapabilityKind::InferenceModel, name: "llama".to_owned(), site: site.to_owned(), + stable_id: None, } } @@ -1376,6 +1378,18 @@ mod tests { assert_eq!(sites(&snapshot.load()), ["east", "west"], "the rewrite applied"); eventually("west polled", || peers.fetches("west") > 0); + write(&yaml(&[])); + eventually("the no-route revision applied", || counts().applied() == 2); + assert!(snapshot.load().candidates.is_empty(), "the final withdrawal is serving"); + + write("local_site: [not, a, site\n"); + eventually("the malformed revision rejected", || counts().rejected() == 2); + assert!(snapshot.load().candidates.is_empty(), "malformed updates retain the no-route revision"); + + write(&yaml(&["east"])); + eventually("the restored route applied", || counts().applied() == 3); + assert_eq!(sites(&snapshot.load()), ["east"], "restoration resumes routing"); + drop(grid); std::fs::remove_file(&path).expect("cleanup"); } diff --git a/gateway/ai-grid-filters/src/descriptor.rs b/gateway/ai-grid-filters/src/descriptor.rs index 94e241957..19ef43535 100644 --- a/gateway/ai-grid-filters/src/descriptor.rs +++ b/gateway/ai-grid-filters/src/descriptor.rs @@ -206,6 +206,7 @@ pub(crate) fn default_stable_id(kind: CapabilityKind, name: &str, site: &str, cl /// [`FilterError`] if the list exceeds [`MAX_CANDIDATES`], any /// name/site/cluster field is blank or oversized, or a duplicate /// (kind, name, site, cluster) tuple exists. +#[cfg(test)] pub(crate) fn validate_candidates(raw: Vec) -> Result, FilterError> { validate_candidates_with_empty(raw, false) } diff --git a/gateway/ai-grid-filters/src/serving.rs b/gateway/ai-grid-filters/src/serving.rs index fcdb946c2..0b649a5d6 100644 --- a/gateway/ai-grid-filters/src/serving.rs +++ b/gateway/ai-grid-filters/src/serving.rs @@ -331,6 +331,7 @@ pub fn spawn_grid_routing(config: &GridServingConfig) -> Result, } +/// One upstream cluster. #[derive(Deserialize)] struct BackendCluster { + /// Cluster identifier. name: String, + /// TLS settings, absent for plaintext. tls: Option, } +/// TLS properties required for an authenticated provider hop. #[derive(Deserialize)] struct BackendTls { + /// Expected server name. sni: String, + /// Certificate verification switch. verify: bool, + /// Trusted CA bundle. ca: Option, + /// Mutual-TLS client identity. client_cert: Option, } +/// CA trust input used by the load balancer. #[derive(Deserialize)] struct BackendCa { + /// CA certificate path. ca_path: String, } +/// Client identity used by the load balancer. #[derive(Deserialize)] struct BackendClientCert { + /// Client certificate path. cert_path: String, + /// Client private key path. key_path: String, } @@ -272,12 +286,20 @@ fn provider_hop_backends(config: &Config) -> Result, pr let mut backends = BTreeMap::new(); let mut unverified = BTreeSet::new(); for chain in &config.filter_chains { - if !chain.filters.iter().any(|filter| filter.filter_type == "grid_site_route") { + if !chain + .filters + .iter() + .any(|filter| filter.filter_type == "grid_site_route") + { continue; } - for filter in chain.filters.iter().filter(|filter| filter.filter_type == "load_balancer") { - let parsed: LoadBalancerBackends = serde_yaml::from_value(filter.config.clone()) - .map_err(|error| -> praxis_filter::FilterError { + for filter in chain + .filters + .iter() + .filter(|filter| filter.filter_type == "load_balancer") + { + let parsed: LoadBalancerBackends = + serde_yaml::from_value(filter.config.clone()).map_err(|error| -> praxis_filter::FilterError { format!("grid: parsing load_balancer backends: {error}").into() })?; for backend in parsed.clusters { @@ -333,6 +355,7 @@ fn config_arg>(args: I) -> Result, } #[cfg(test)] +#[expect(clippy::expect_used, reason = "test fixtures use checked parsing")] mod tests { use std::ffi::OsStr; From fcbf30025d6a7d54e720a2d9458fe07d953111e8 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sun, 4 Oct 2026 18:13:37 +0000 Subject: [PATCH 07/10] fix(operator): align no-route integration with current main Signed-off-by: Brent Salisbury --- operator/src/controller/grid_network.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 6dfbdeee6..562b42734 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -4454,10 +4454,6 @@ fn log_capacity_changes(logged: &ChangeLog, network: &str, providers: &[Inferenc /// advances on spec and status writes, and falls back to `metadata.generation` /// when no parseable resource version is present. Equal revisions break ties /// via `writer_id`, which is the advertising SWIM site identity. -#[expect( - clippy::too_many_lines, - reason = "the provider-to-CRDT mapping keeps every advertised field derived from one source object" -)] fn provider_state_from_kube( provider: &InferenceProvider, network_id: &str, From c1e474fa384fd76ed4a2634d7e39b38bb37a8dec Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Sun, 4 Oct 2026 18:34:58 +0000 Subject: [PATCH 08/10] style(gateway): apply nightly rustfmt to no-route integration Signed-off-by: Brent Salisbury --- gateway/ai-grid-filters/src/control.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index da30c501a..a894b145b 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -1384,7 +1384,10 @@ mod tests { write("local_site: [not, a, site\n"); eventually("the malformed revision rejected", || counts().rejected() == 2); - assert!(snapshot.load().candidates.is_empty(), "malformed updates retain the no-route revision"); + assert!( + snapshot.load().candidates.is_empty(), + "malformed updates retain the no-route revision" + ); write(&yaml(&["east"])); eventually("the restored route applied", || counts().applied() == 3); From 8e1f40d0785eb65c9fd1947b97109adf5c1a36ac Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Thu, 8 Oct 2026 17:03:02 +0000 Subject: [PATCH 09/10] fix(grid): complete no-route integration and guard backend reload Signed-off-by: Brent Salisbury --- .../templates/crds/gridnetwork.yaml | 45 ++- charts/praxis-gateway/README.md | 15 +- deploy/crds/gridnetwork.yaml | 45 ++- docs/architecture/consumer-config.md | 38 +-- gateway/Cargo.lock | 1 + gateway/Cargo.toml | 7 +- gateway/ai-grid-filters/Cargo.toml | 4 +- gateway/ai-grid-filters/benches/choose.rs | 2 +- gateway/ai-grid-filters/src/control.rs | 64 ++-- gateway/ai-grid-filters/src/descriptor.rs | 12 +- gateway/ai-grid-filters/src/route.rs | 119 +++++++- gateway/ai-grid-filters/src/serving.rs | 7 +- gateway/ai-grid-filters/src/snapshot.rs | 26 +- gateway/src/main.rs | 113 +++++-- gateway/tests/serving_reload.rs | 140 ++++++++- operator/src/controller/grid_network.rs | 77 ++++- operator/src/controller/inference_provider.rs | 28 +- operator/src/crd/grid_network.rs | 4 + operator/src/resources/consumer_config.rs | 281 ++++++++++++++---- operator/src/resources/serving_config.rs | 79 +++-- .../grid-provider-traffic/README.md | 10 +- xtask/src/env/certs.rs | 8 +- xtask/src/env/image_overrides.rs | 20 +- 23 files changed, 911 insertions(+), 234 deletions(-) diff --git a/charts/grid-operator/templates/crds/gridnetwork.yaml b/charts/grid-operator/templates/crds/gridnetwork.yaml index 72c963b5b..74cb05ab2 100644 --- a/charts/grid-operator/templates/crds/gridnetwork.yaml +++ b/charts/grid-operator/templates/crds/gridnetwork.yaml @@ -357,6 +357,18 @@ spec: x-kubernetes-validations: - message: deploymentName is required when mount reconciliation is enabled rule: '!has(self.enabled) || !self.enabled || has(self.deploymentName)' + supportsProjectedCredentials: + default: false + description: |- + Declare that the running consumer has the dynamic `credential_inject` + filter configured with the projected Secret mount root. This is a + fail-closed capability gate for credential-bearing overlay revisions. + Set it only after the consumer has loaded a config containing that + filter; the Grid operator does not own or roll out the consumer + Deployment. + + Default: `false`. + type: boolean telemetry: description: |- Optional OpenTelemetry exporter and sampling settings for this gateway. @@ -472,21 +484,46 @@ spec: transport: description: TLS mode and SNI configured for that embedded gateway upstream. properties: + caSecretRef: + description: |- + Optional gateway-namespace Secret supplying a custom CA bundle for `tls` transport. + Specify only its name and optional key; the Secret must be in the namespace + named by the containing `GatewayRef`. The mounted path is + `/run/secrets/grid-backend-ca/{secret-name}/{key}`; the key defaults to + `ca.crt`. Omitted uses the process trust store. + nullable: true + properties: + key: + description: Optional key containing the CA bundle. Defaults to `ca.crt`. + minLength: 1 + nullable: true + type: string + name: + description: Secret name in the target gateway namespace. + minLength: 1 + type: string + required: + - name + type: object mode: - description: 'Transport mode: `mutual_tls` or `plaintext`.' + description: 'Transport mode: `mutual_tls`, `tls`, or explicit `plaintext`.' enum: - mutual_tls + - tls - plaintext type: string sni: - description: |- - TLS Server Name Indication (required when mode is `mutual_tls`; - must not be set when mode is `plaintext`). + description: TLS Server Name Indication, required when mode is `mutual_tls` or `tls`. nullable: true type: string required: - mode type: object + x-kubernetes-validations: + - message: caSecretRef is only valid for tls transport + rule: self.mode == 'tls' || !has(self.caSecretRef) || self.caSecretRef == null + - message: an explicitly empty sni is only valid for plaintext transport + rule: self.mode == 'plaintext' || !has(self.sni) || self.sni.size() > 0 required: - cluster - transport diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index 5d05c08b1..d2b3635e4 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -486,10 +486,10 @@ Before enabling delegation on an existing release, set Grid serving) `consumerConfig.tlsCertMountPath` to paths that do not overlap the current chart mounts. The operator stages its new mounts at those paths, switches to a matching generated config in the same Pod-template update, and -leaves the old Helm mounts in place. **For populated routes, this handoff is not -yet supported with the chart-default AI 0.4.0 image.** The current generated -inline candidates contain `admission_state` and `selection_group`, which the -tested Praxis image rejects. Wait for [Grid #270](https://github.com/praxis-proxy/grid/pull/270), +leaves the old Helm mounts in place. Older generated configs embedded +`admission_state` and `selection_group` in inline candidates, which the +previously supported Praxis AI image rejected. Populated routes require a +Praxis AI image that accepts Grid's versioned-overlay config. Wait for [Grid #270](https://github.com/praxis-proxy/grid/pull/270), a compatible image containing [Praxis AI #1539](https://github.com/praxis-proxy/ai/pull/1539), and qualification of the unmodified generated config before enabling this handoff for populated routes. A `Ready` mount status alone does not prove that @@ -554,8 +554,11 @@ Known limits: - `grid_site_route` does not check provider health, so it can pick a site whose provider gateway is down. That request fails rather than failing over. -- The gateway matches a candidate's cluster to `gatewayConfig.backends` by name only. - Nothing checks that the backend serves the candidate's site. +- Ordinary routing matches a candidate's cluster to `gatewayConfig.backends` by + name; it does not attest which models that backend serves. Provider-hop + headers are allowed only when the declared cluster has a unique verified + mutual-TLS backend with the matching SNI. Same-named plaintext or mismatched + backends fail startup. - The serving watcher does not add load-balancer clusters to `praxis.yaml`. Add each new candidate's backend there too. Listener changes and serving `window_secs` changes require a restart. diff --git a/deploy/crds/gridnetwork.yaml b/deploy/crds/gridnetwork.yaml index 4eda4c386..bd746b9a4 100644 --- a/deploy/crds/gridnetwork.yaml +++ b/deploy/crds/gridnetwork.yaml @@ -349,6 +349,18 @@ spec: x-kubernetes-validations: - message: deploymentName is required when mount reconciliation is enabled rule: '!has(self.enabled) || !self.enabled || has(self.deploymentName)' + supportsProjectedCredentials: + default: false + description: |- + Declare that the running consumer has the dynamic `credential_inject` + filter configured with the projected Secret mount root. This is a + fail-closed capability gate for credential-bearing overlay revisions. + Set it only after the consumer has loaded a config containing that + filter; the Grid operator does not own or roll out the consumer + Deployment. + + Default: `false`. + type: boolean telemetry: description: |- Optional OpenTelemetry exporter and sampling settings for this gateway. @@ -464,21 +476,46 @@ spec: transport: description: TLS mode and SNI configured for that embedded gateway upstream. properties: + caSecretRef: + description: |- + Optional gateway-namespace Secret supplying a custom CA bundle for `tls` transport. + Specify only its name and optional key; the Secret must be in the namespace + named by the containing `GatewayRef`. The mounted path is + `/run/secrets/grid-backend-ca/{secret-name}/{key}`; the key defaults to + `ca.crt`. Omitted uses the process trust store. + nullable: true + properties: + key: + description: Optional key containing the CA bundle. Defaults to `ca.crt`. + minLength: 1 + nullable: true + type: string + name: + description: Secret name in the target gateway namespace. + minLength: 1 + type: string + required: + - name + type: object mode: - description: 'Transport mode: `mutual_tls` or `plaintext`.' + description: 'Transport mode: `mutual_tls`, `tls`, or explicit `plaintext`.' enum: - mutual_tls + - tls - plaintext type: string sni: - description: |- - TLS Server Name Indication (required when mode is `mutual_tls`; - must not be set when mode is `plaintext`). + description: TLS Server Name Indication, required when mode is `mutual_tls` or `tls`. nullable: true type: string required: - mode type: object + x-kubernetes-validations: + - message: caSecretRef is only valid for tls transport + rule: self.mode == 'tls' || !has(self.caSecretRef) || self.caSecretRef == null + - message: an explicitly empty sni is only valid for plaintext transport + rule: self.mode == 'plaintext' || !has(self.sni) || self.sni.size() > 0 required: - cluster - transport diff --git a/docs/architecture/consumer-config.md b/docs/architecture/consumer-config.md index 1061c43a7..296584370 100644 --- a/docs/architecture/consumer-config.md +++ b/docs/architecture/consumer-config.md @@ -74,9 +74,9 @@ behavior. When `spec.gatewayRefs[].consumerConfig.enabled: true`, the `GridNetwork` controller renders a `praxis.yaml`-keyed `ConfigMap` in the gateway namespace -when consumer configuration renders successfully. An empty candidate overlay -produces a fail-closed 503-only config without route filters. For nonempty -candidates, the generated config includes: +when consumer configuration renders successfully. The generated config keeps +the watched route filter and complete endpoint inventory even when the current +overlay has no candidates, so a later revision can restore routing. **Validation status:** `verify-api-fallback-native` proves end-to-end runtime consumption of the operator-generated `ConfigMap`. The xtask harness reads the @@ -202,9 +202,8 @@ an empty list in arbitrary startup YAML: Grid publishes authoritative empty revisions without a capability flag. Every consumer of a gateway's overlay must therefore run an image with empty-snapshot -support before this Grid version is deployed. The chart-default Praxis AI 0.4.0 -image is not compatible; the paired AI change and a compatible image release -are prerequisites for the next Grid release. +support before this Grid version is deployed. The paired AI change and a +compatible image release are prerequisites for the next Grid release. The embedded gateway's filter chain and upstream cluster definitions remain in its startup Praxis configuration, but its changing candidate list is not a @@ -272,9 +271,11 @@ one patch. Old pods keep their previous config and projections during rollout. Grid waits until no old replicas remain before reusing the inactive slot or pruning obsolete mounts. This also lets a new Deployment become ready before Grid replaces the bootstrap config. This verifies the Kubernetes rollout, not -Praxis acceptance of populated routes. The current generated inline candidates -include `admission_state` and `selection_group`, which the tested Praxis image -rejects. Populated-route use requires the versioned-overlay config in +Praxis acceptance of populated routes. Older generated configs embedded +`admission_state` and `selection_group` in inline candidates, which the +previously supported Praxis AI image rejected. Populated routes require a +Praxis AI image that accepts Grid's versioned-overlay config. They also require +the versioned-overlay config in [Grid #270](https://github.com/praxis-proxy/grid/pull/270), a compatible image containing [Praxis AI #1539](https://github.com/praxis-proxy/ai/pull/1539), and an unmodified generated-config request probe. Until then, do not release @@ -291,11 +292,11 @@ Secret reference and key changes. When a reference is removed, the old mount remains until pods with the new config are ready, then Grid removes only mounts recorded as Grid-owned. If the last provider disappears, the operator distributes an empty authoritative -routing overlay and renders the generated consumer config as a 503-only response -with no `intelligent_route` filter. For delegated mounts, the empty overlay is -published before the 503-only config is reconciled, so consumers using the -watched overlay stop selecting providers even if config reconciliation fails. -A static-only consumer still needs the 503-only config to apply and reload. +routing overlay. The generated consumer config retains `intelligent_route`, its +overlay watcher, and the complete endpoint inventory so a later revision can +restore routing. For delegated mounts, the empty overlay is published before +config reconciliation, so consumers using the watched overlay stop selecting +providers even if config reconciliation fails. Obsolete Grid-owned mounts are pruned only after the matching config rollout is ready. Other Deployment fields, containers, volumes, mounts, and Helm resources are preserved. @@ -451,11 +452,10 @@ overlay is a separate file that `intelligent_route` validates and reloads. See [Reload and rollout](#reload-and-rollout) below. When no inference candidates remain, the operator distributes an empty -authoritative routing overlay and updates its generated consumer `ConfigMap` to -a 503-only response with no `intelligent_route` filter. The ConfigMap update -does not itself restart gateway pods; use a supported file reload or restart -when the running gateway does not reload it. The empty overlay also fails closed -for routes already using the dynamic overlay. +authoritative routing overlay. The generated consumer `ConfigMap` retains the +watched `intelligent_route` filter and configured endpoints for restoration. +The ConfigMap update does not itself restart gateway pods; use a supported file +reload or restart when the running gateway does not reload it. ## Edge-ingress deployments diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 8ff66e8f2..8dcd6441e 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -1520,6 +1520,7 @@ dependencies = [ "praxis-proxy-protocol", "praxis-proxy-tls", "rustls", + "serde", "tokio", "tokio-rustls", "tracing", diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index 2726ea663..08695fb8f 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -7,6 +7,10 @@ edition = "2024" rust-version = "1.96" license = "Apache-2.0" +[workspace.dependencies] +serde = { version = "1.0.228", features = ["derive"] } +serde_yaml = { version = "0.10.7", package = "yaml_serde" } + [workspace.lints.rust] dead_code = "deny" unsafe_code = "forbid" @@ -321,12 +325,13 @@ praxis-tls = { version = "0.7.3", package = "praxis-proxy-tls" } tokio = { version = "1.53.1", features = ["io-util", "net", "rt", "sync", "time"] } tokio-rustls = { version = "0.26.5", default-features = false, features = ["logging", "tls12"] } httparse = "1.10.1" +serde.workspace = true +serde_yaml.workspace = true [dev-dependencies] # The reload test runs a real mTLS signals peer against the binary. certs = { path = "../certs", default-features = false, features = ["verifier"] } rustls = { version = "0.23.45", features = ["ring"] } -serde_yaml = { version = "0.10", package = "yaml_serde" } tokio = { version = "1.53.1", features = ["macros"] } # machete can't see these: used as `praxis` / `praxis_core` / `praxis_filter`, not their package names. diff --git a/gateway/ai-grid-filters/Cargo.toml b/gateway/ai-grid-filters/Cargo.toml index fc665d808..3a262efd7 100644 --- a/gateway/ai-grid-filters/Cargo.toml +++ b/gateway/ai-grid-filters/Cargo.toml @@ -16,10 +16,10 @@ grid-signals = { path = "../../signals" } # poller and cert issuance are real dependencies, not just test ones. grid-signals-client = { path = "../../signals-client", default-features = false, features = ["fips"] } certs = { path = "../../certs", default-features = false, features = ["verifier", "fips"] } -serde = { version = "1", features = ["derive"] } +serde.workspace = true # praxis-proxy-filter 0.7 renamed its YAML crate to the yaml_serde fork and # aliases it as serde_yaml; match it so filter config types line up. -serde_yaml = { version = "0.10", package = "yaml_serde" } +serde_yaml.workspace = true arc-swap = "1" async-trait = "0.1" tracing = "0.1" diff --git a/gateway/ai-grid-filters/benches/choose.rs b/gateway/ai-grid-filters/benches/choose.rs index e36f1566a..2b2d825ea 100644 --- a/gateway/ai-grid-filters/benches/choose.rs +++ b/gateway/ai-grid-filters/benches/choose.rs @@ -62,7 +62,7 @@ fn snapshot(sites: u32) -> RouteSnapshot { fn filter(snapshot: RouteSnapshot) -> Result, FilterError> { let mut config = load_serving_config("testdata/serving-config.json")?; config.peers.clear(); - let runtime = spawn_grid_routing(&config)?; + let runtime = spawn_grid_routing(&config, std::collections::BTreeMap::new())?; let mut registry = FilterRegistry::with_builtins(); let snapshot = Arc::new(ArcSwap::from_pointee(snapshot)); register_grid_filters( diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index a894b145b..a081cc1f0 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -17,7 +17,7 @@ use grid_signals_client::{PollHandle, PollerConfig}; use praxis_filter::FilterError; use crate::{ - descriptor::{RouteCandidate, validate_candidates, validate_local_site}, + descriptor::{RouteCandidate, validate_local_site, validate_provider_hop_clusters, validate_serving_candidates}, health::ClusterHealth, pin::TagKey, prefix::{AffinitySettings, PrefixAffinity}, @@ -102,12 +102,22 @@ impl Topology { /// Validate the topology half of `config`. fn from_config(config: &GridServingConfig) -> Result { validate_local_site(&config.local_site)?; - let base = validate_candidates(config.candidates.clone())?; + let base = validate_serving_candidates(config.candidates.clone())?; + let provider_hop_clusters = validate_provider_hop_clusters(config.provider_hop_clusters.clone())?; + for candidate in &config.candidates { + if provider_hop_clusters.contains(&candidate.cluster) && candidate.stable_id.is_none() { + return Err(format!( + "grid: candidate '{}' on provider-hop cluster '{}' is missing stable_id", + candidate.name, candidate.cluster + ) + .into()); + } + } Ok(Self { base: Arc::from(base), local_site: Arc::from(config.local_site.as_str()), load_window_ms: config.load_window_ms, - provider_hop_clusters: validate_provider_hop_clusters(config.provider_hop_clusters.clone())?, + provider_hop_clusters, }) } @@ -122,7 +132,7 @@ impl Topology { store: &LoadStore, now: i64, health: &ClusterHealth, - shedding: &std::collections::BTreeSet>, + shedding: &BTreeSet>, gauged: &mut Gauged, availability: &AvailabilitySettings, ) -> RouteSnapshot { @@ -133,11 +143,12 @@ impl Topology { availability, learned: &mut gauged.learned, }; - let ordered = RouteSnapshot::from_store( + let mut ordered = RouteSnapshot::from_store( self.base.iter().cloned().collect(), Arc::clone(&self.local_site), &mut inputs, ); + ordered.provider_hop_clusters = Arc::new(self.provider_hop_clusters.clone()); // Praxis demotes a cluster only once it has reported health. Without a registry the // gateway knows nothing about backends, so it demotes nothing rather than guessing. let ordered = if health.observed() { @@ -224,6 +235,9 @@ pub(crate) struct Control { /// The tuning generation the applied config adopted. tuned: u64, + + /// Verified TLS identities from the loaded Praxis backend configuration. + backend_tls: BTreeMap, } impl Control { @@ -243,11 +257,12 @@ impl Control { let topology = Topology::from_config(config)?; // Cold start: config order until the first poll re-orders it by live load. let mut gauged = Gauged::new(); - let cold_start = RouteSnapshot::from_static( + let mut cold_start = RouteSnapshot::from_static( topology.base.iter().cloned().collect(), Arc::clone(&topology.local_site), ) .published(&mut gauged.published); + cold_start.provider_hop_clusters = Arc::new(topology.provider_hop_clusters.clone()); Ok(Self { store: Arc::new(LoadStore::with_combine( Duration::from_secs(config.window_secs), @@ -265,6 +280,7 @@ impl Control { health: Arc::default(), tuning: Arc::default(), tuned: 0, + backend_tls, }) } @@ -341,6 +357,7 @@ impl Control { return Ok(None); } // Validated before any poller starts, so an invalid config starts and drops nothing. + validate_provider_hop_binding(config, &self.backend_tls)?; let topology = Arc::new(validate_config(config)?); let tag_key = load_tag_key(&self.tuning.affinity())?; if config.window_secs != self.window_secs { @@ -477,17 +494,8 @@ pub(crate) fn load_tag_key(affinity: &AffinitySettings) -> Result /// The topology of `config`, refusing what no retry can fix: a bad candidate, peer, or duplicate site. fn validate_config(config: &GridServingConfig) -> Result { validate_local_site(&config.local_site)?; - validate_candidates(config.candidates.clone())?; + validate_serving_candidates(config.candidates.clone())?; let topology = Topology::from_config(config)?; - for candidate in &config.candidates { - if topology.provider_hop_clusters.contains(&candidate.cluster) && candidate.stable_id.is_none() { - return Err(format!( - "grid: candidate '{}' on provider-hop cluster '{}' is missing stable_id", - candidate.name, candidate.cluster - ) - .into()); - } - } let mut sites = std::collections::HashSet::with_capacity(config.peers.len()); for peer in &config.peers { validate_peer(peer)?; @@ -498,6 +506,28 @@ fn validate_config(config: &GridServingConfig) -> Result Ok(topology) } +/// Bind every declared provider hop to one verified backend and its exact TLS SNI. +fn validate_provider_hop_binding( + config: &GridServingConfig, + backend_tls: &BTreeMap, +) -> Result<(), FilterError> { + let clusters = validate_provider_hop_clusters(config.provider_hop_clusters.clone())?; + if config.provider_hop_sni.len() != clusters.len() { + return Err("grid: provider-hop SNI declarations must match the allowlist".into()); + } + for cluster in &clusters { + let declared = config + .provider_hop_sni + .get(cluster) + .filter(|sni| !sni.trim().is_empty()) + .ok_or_else(|| -> FilterError { format!("grid: missing provider-hop SNI for {cluster}").into() })?; + if backend_tls.get(cluster) != Some(declared) { + return Err(format!("grid: provider-hop backend {cluster} lacks matching verified TLS identity").into()); + } + } + Ok(()) +} + /// Order the current topology under the current availability tuning and swap it in. /// /// Every refresher orders under `swap`, so the last store always reflects the latest @@ -538,7 +568,7 @@ where /// /// Each file is hashed on its own, so no buffer holds the concatenated key. fn identity_digest(config: &GridServingConfig, affinity: &AffinitySettings) -> [u8; 32] { - let paths: std::collections::BTreeSet<&str> = config + let paths: BTreeSet<&str> = config .peers .iter() .flat_map(|peer| [&peer.grid_ca_path, &peer.client_cert_path, &peer.client_key_path]) diff --git a/gateway/ai-grid-filters/src/descriptor.rs b/gateway/ai-grid-filters/src/descriptor.rs index 19ef43535..84d12b9bf 100644 --- a/gateway/ai-grid-filters/src/descriptor.rs +++ b/gateway/ai-grid-filters/src/descriptor.rs @@ -223,7 +223,13 @@ pub(crate) fn validate_serving_candidates(raw: Vec) -> Result) -> Result, FilterError> { +fn validate_candidates_with_empty( + raw: Vec, + allow_empty: bool, +) -> Result, FilterError> { + if raw.is_empty() && !allow_empty { + return Err("grid: candidates must not be empty outside a versioned serving config".into()); + } if raw.len() > MAX_CANDIDATES { return Err(format!("grid: candidates exceeds maximum of {MAX_CANDIDATES}").into()); } @@ -433,8 +439,8 @@ mod tests { #[test] fn empty_candidates_are_an_authoritative_no_route_snapshot() { - let candidates = validate_candidates(vec![]).unwrap_or_else(|_| std::process::abort()); - assert!(candidates.is_empty()); + let candidates = validate_serving_candidates(vec![]).expect("versioned serving can withdraw every candidate"); + assert!(candidates.is_empty(), "the empty serving revision is authoritative"); } #[test] diff --git a/gateway/ai-grid-filters/src/route.rs b/gateway/ai-grid-filters/src/route.rs index acaa4da35..e41838778 100644 --- a/gateway/ai-grid-filters/src/route.rs +++ b/gateway/ai-grid-filters/src/route.rs @@ -49,6 +49,53 @@ const PROVIDER_HOP_REQUEST_ID_HEADER: &str = "x-ai-routing-request-id"; /// Internal request header carrying the serving overlay revision. const OVERLAY_REVISION_HEADER: &str = "x-ai-routing-revision"; +/// Whether the current serving revision permits a previous filter's choice. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum RouteDecision { + /// An authoritative empty revision withdraws every route. + NoRoute, + /// Another filter already chose an upstream. + KeepEarlier, + /// This filter should select a candidate. + Select, +} + +/// Resolve the empty revision before preserving a prior filter's selection. +fn route_decision(snapshot: &RouteSnapshot, _model: &str, earlier_selected: bool) -> RouteDecision { + if snapshot.candidates.is_empty() { + RouteDecision::NoRoute + } else if earlier_selected { + RouteDecision::KeepEarlier + } else { + RouteDecision::Select + } +} + +/// Set hop context only for a candidate whose backend TLS identity remains verified across reloads. +fn write_provider_context( + ctx: &mut HttpFilterContext<'_>, + candidate: &RouteCandidate, + snapshot: &RouteSnapshot, +) -> Result<(), FilterError> { + if !snapshot.provider_hop_clusters.contains(candidate.cluster.as_ref()) { + return Ok(()); + } + let candidate_id = http::HeaderValue::from_str(&candidate.stable_id) + .map_err(|error| -> FilterError { format!("grid: invalid provider-hop candidate ID: {error}").into() })?; + let request_id = ctx.id_generator.generate(ctx.time_source); + let request_id = http::HeaderValue::from_str(&request_id) + .map_err(|error| -> FilterError { format!("grid: invalid provider-hop request ID: {error}").into() })?; + ctx.request_headers_to_set.push(( + http::header::HeaderName::from_static(SELECTED_CANDIDATE_HEADER), + candidate_id, + )); + ctx.request_headers_to_set.push(( + http::header::HeaderName::from_static(PROVIDER_HOP_REQUEST_ID_HEADER), + request_id, + )); + Ok(()) +} + /// Default request header carrying the model name. fn default_model_header() -> String { "X-Model".to_owned() @@ -271,7 +318,13 @@ impl GridSiteRouteFilter { /// Send the request to `pick`: count the decision, select its cluster, and record its prefix /// there for the response to confirm or withdraw. - fn route(&self, ctx: &mut HttpFilterContext<'_>, pick: &Pick<'_>, keys: Option) { + fn route( + &self, + ctx: &mut HttpFilterContext<'_>, + snapshot: &RouteSnapshot, + pick: &Pick<'_>, + keys: Option, + ) -> Result<(), FilterError> { pick.decisions.record(pick.fallback); let candidate = pick.candidate; let mut responses = false; @@ -288,6 +341,8 @@ impl GridSiteRouteFilter { ctx.request_headers_to_remove.push(http::header::ACCEPT_ENCODING); } ctx.cluster = Some(Arc::clone(&candidate.cluster)); + write_provider_context(ctx, candidate, snapshot)?; + Ok(()) } /// Route a request that names stored state to the site that stored it. @@ -295,25 +350,32 @@ impl GridSiteRouteFilter { /// `None` for a request that names none. A site no longer in the grid gets /// 404, since the state lives only there. One that admits no new request now /// gets 503. - fn pin(&self, ctx: &mut HttpFilterContext<'_>, model: Option<&str>) -> Option { + #[expect( + clippy::too_many_lines, + reason = "pinned-state outcomes and hop context are handled together" + )] + fn pin(&self, ctx: &mut HttpFilterContext<'_>, model: Option<&str>) -> Result, FilterError> { let snapshot = self.snapshot.load(); let key = self.affinity.tag_key.load(); - let (holder, rewrite) = match pinned_site(ctx, &snapshot, key.as_deref())? { + let Some(pinned) = pinned_site(ctx, &snapshot, key.as_deref()) else { + return Ok(None); + }; + let (holder, rewrite) = match pinned { Ok(pinned) => pinned, - Err(answer) => return Some(answer), + Err(answer) => return Ok(Some(answer)), }; let site = holder.as_ref().map(|holder| &holder.site); let candidate = match pinned_target(&snapshot, holder.as_ref(), model) { Target::Found(candidate) => candidate, Target::Gone => { tracing::debug!(site = ?site, "grid_site_route: the cluster holding the state left the grid"); - return Some(FilterAction::Reject(Rejection::status(404))); + return Ok(Some(FilterAction::Reject(Rejection::status(404)))); }, Target::Unavailable => { tracing::debug!(site = ?site, "grid_site_route: the cluster holding the state admits no new request"); - return Some(FilterAction::Reject( + return Ok(Some(FilterAction::Reject( Rejection::status(503).with_header("retry-after", RETRY_AFTER_SECS), - )); + ))); }, }; let cluster = Arc::clone(&candidate.cluster); @@ -322,10 +384,11 @@ impl GridSiteRouteFilter { state.served = Some((Arc::clone(&candidate.site), Arc::clone(&candidate.cluster))); }); ctx.cluster = Some(cluster); + write_provider_context(ctx, candidate, &snapshot)?; ctx.rewritten_path = rewrite.or_else(|| ctx.rewritten_path.take()); // Ids are tagged in the body, so it must come back uncompressed. ctx.request_headers_to_remove.push(http::header::ACCEPT_ENCODING); - Some(FilterAction::Continue) + Ok(Some(FilterAction::Continue)) } /// A tagger for the ids of the answer `state` was served by. @@ -438,11 +501,27 @@ impl HttpFilter for GridSiteRouteFilter { Ok(FilterAction::Continue) } + #[expect( + clippy::too_many_lines, + reason = "one snapshot decides withdrawal, pinned state, and normal selection" + )] async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { self.health.observe(ctx.health_registry); - // An earlier selecting filter wins; never override its choice. - if ctx.cluster.is_some() || ctx.upstream.is_some() { - return Ok(FilterAction::Continue); + for name in [ + SELECTED_CANDIDATE_HEADER, + PROVIDER_HOP_REQUEST_ID_HEADER, + OVERLAY_REVISION_HEADER, + ] { + ctx.request_headers_to_remove + .push(http::header::HeaderName::from_static(name)); + } + let snapshot = self.snapshot.load(); + match route_decision(&snapshot, "", ctx.cluster.is_some() || ctx.upstream.is_some()) { + RouteDecision::NoRoute => { + return Ok(refuse(Refused::NoRoute, "", self.turn.fetch_add(1, Ordering::Relaxed))); + }, + RouteDecision::KeepEarlier => return Ok(FilterAction::Continue), + RouteDecision::Select => {}, } // A copy of the request reference, so the model outlives writes to ctx. let request = ctx.request; @@ -450,7 +529,7 @@ impl HttpFilter for GridSiteRouteFilter { .headers .get(&self.model_header) .and_then(|value| value.to_str().ok()); - if let Some(action) = self.pin(ctx, model) { + if let Some(action) = self.pin(ctx, model)? { return Ok(action); } let Some(model) = model else { @@ -460,14 +539,13 @@ impl HttpFilter for GridSiteRouteFilter { return Ok(unrouted(400)); }; - let snapshot = self.snapshot.load(); let keys = ctx .get_filter_state_mut::() .and_then(|state| state.keys.take()) .map(|keys| keys.for_model(model)); match self.choose(&snapshot, model, keys.as_ref()) { Ok(pick) => { - self.route(ctx, &pick, keys); + self.route(ctx, &snapshot, &pick, keys)?; Ok(FilterAction::Continue) }, Err(answer) => Ok(answer), @@ -656,6 +734,7 @@ fn pinned_target<'snap>(snapshot: &'snap RouteSnapshot, holder: Option<&Holder>, } /// Count and answer a request no candidate took. +#[expect(clippy::too_many_lines, reason = "each refusal maps to a distinct protocol response")] fn refuse(reason: Refused, model: &str, turn: usize) -> FilterAction { tracing::debug!(model = %model, reason = ?reason, "grid_site_route: no admitted candidate"); reason.record(); @@ -676,7 +755,14 @@ fn refuse(reason: Refused, model: &str, turn: usize) -> FilterAction { "capacity_exhausted", "every site serving this model is at capacity", ), - Refused::NotReady | Refused::NoRoute | Refused::NoModel => retry_later( + Refused::NoRoute => error_response( + 404, + "invalid_request_error", + "no_route", + "no route serves this model", + None, + ), + Refused::NotReady | Refused::NoModel => retry_later( turn, 503, "server_error", @@ -1147,6 +1233,7 @@ mod tests { sorted .iter() .map(|(site, ..)| CandidateConfig { + stable_id: None, cluster: format!("pool-{site}"), credential: None, admission: AdmissionState::default(), @@ -1167,6 +1254,7 @@ mod tests { candidates, local_site: Arc::from("a"), shedding: BTreeSet::new(), + provider_hop_clusters: Arc::default(), } } @@ -1274,6 +1362,7 @@ mod tests { [("llama", "pool-llama"), ("granite", "pool-granite")] .into_iter() .map(|(name, cluster)| CandidateConfig { + stable_id: None, cluster: cluster.to_owned(), credential: None, admission: AdmissionState::default(), diff --git a/gateway/ai-grid-filters/src/serving.rs b/gateway/ai-grid-filters/src/serving.rs index 0b649a5d6..3220cd3e0 100644 --- a/gateway/ai-grid-filters/src/serving.rs +++ b/gateway/ai-grid-filters/src/serving.rs @@ -7,7 +7,7 @@ //! resolved order. A watch on the config file applies the operator's rewrites. use std::{ - collections::{BTreeMap, BTreeSet}, + collections::BTreeMap, fs, path::PathBuf, sync::{Arc, Mutex, PoisonError}, @@ -319,7 +319,10 @@ pub fn load_serving_config(path: &str) -> Result /// Returns [`FilterError`] if the local site or candidate topology is invalid, a /// peer's certificate material cannot be read or parsed, or a poller thread /// cannot be spawned. -pub fn spawn_grid_routing(config: &GridServingConfig) -> Result { +pub fn spawn_grid_routing( + config: &GridServingConfig, + backend_tls: BTreeMap, +) -> Result { // The peer scrapers load TLS here, before the server installs the provider. praxis_tls::provider::install(); let start = Box::new(|peer: &PeerServingConfig, poller: &_, store, refresh| { diff --git a/gateway/ai-grid-filters/src/snapshot.rs b/gateway/ai-grid-filters/src/snapshot.rs index c6192995e..9a95880a0 100644 --- a/gateway/ai-grid-filters/src/snapshot.rs +++ b/gateway/ai-grid-filters/src/snapshot.rs @@ -42,6 +42,9 @@ pub struct RouteSnapshot { /// Models answering 503 because every healthy site serving them is past full. pub shedding: BTreeSet>, + + /// Verified provider-hop clusters from the same serving revision as the candidates. + pub provider_hop_clusters: Arc>, } /// Per model while shedding is decided: whether every site is measured, and whether each @@ -76,6 +79,7 @@ impl RouteSnapshot { local_site, scores, shedding: BTreeSet::new(), + provider_hop_clusters: Arc::default(), } } @@ -91,23 +95,6 @@ impl RouteSnapshot { candidates: Vec, local_site: Arc, inputs: &mut Inputs<'_, S>, - ) -> Self { - Self::from_store_with_provider_hops(candidates, local_site, store, now_ms, window_ms, BTreeSet::new()) - } - - /// Order candidates by load and keep the provider-hop allowlist in the same - /// immutable snapshot so route and trust decisions change atomically. - #[expect( - clippy::too_many_arguments, - reason = "the immutable snapshot constructor takes all independent routing inputs explicitly" - )] - pub fn from_store_with_provider_hops( - candidates: Vec, - local_site: Arc, - store: &LoadStore, - now_ms: i64, - window_ms: i64, - provider_hop_clusters: BTreeSet, ) -> Self { // Score each candidate once, then sort the pairs: load_of allocates a // store key and scans a window, too costly to repeat inside sort_by. @@ -325,7 +312,9 @@ impl RouteSnapshot { (demoted, candidate) }) .collect(); - Self::ranked(ranked, self.local_site) + let mut ordered = Self::ranked(ranked, self.local_site); + ordered.provider_hop_clusters = self.provider_hop_clusters; + ordered } /// This snapshot, after setting `grid_route_site_score` for each of its site/cluster pairs. @@ -371,6 +360,7 @@ impl RouteSnapshot { scores, local_site, shedding: BTreeSet::new(), + provider_hop_clusters: Arc::default(), } } } diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 123d112a9..cece07a4a 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -12,7 +12,11 @@ //! Kubernetes client stack. See `deploy/gateway/Containerfile` and the //! `gateway-image` make target. -use std::{ffi::OsStr, process::ExitCode}; +use std::{ + collections::{BTreeMap, BTreeSet}, + ffi::OsStr, + process::ExitCode, +}; use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; use serde::Deserialize; @@ -81,6 +85,10 @@ fn preflight() -> Option { None } +#[expect( + clippy::too_many_lines, + reason = "gateway startup assembles dependent runtime stages" +)] fn main() -> ExitCode { if let Some(exit) = preflight() { return exit; @@ -109,18 +117,30 @@ fn main() -> ExitCode { // runtime holding their handles. grid_site_route registers over the snapshot // the pollers refresh. Dropping the runtime stops the pollers, so it is // bound until the server returns. - let grid_runtime = match std::env::var("GRID_SERVING_CONFIG") - .ok() - .map(|path| start_grid_routing(&path, &config, &mut registry)) - { + let (grid_runtime, backend_tls) = match std::env::var("GRID_SERVING_CONFIG").ok().map(|path| { + let backend_tls = provider_hop_backends(&config)?; + let backend_sni = backend_tls + .iter() + .map(|(name, tls)| (name.clone(), tls.sni.clone())) + .collect(); + let runtime = start_grid_routing(&path, backend_sni, &mut registry)?; + Ok::<_, praxis_filter::FilterError>((runtime, backend_tls)) + }) { Some(Err(err)) => return praxis::report_fatal(&err, log_output), - Some(Ok(runtime)) => Some(runtime), - None => None, + Some(Ok((runtime, backend_tls))) => (Some(runtime), Some(backend_tls)), + None => (None, None), }; // Use the returning server path so both the routing runtime and tracing // provider can shut down cleanly after the listeners stop. - let result = praxis::try_run_server_with_registry(config, registry, config_file, log_level); + let mut composition = praxis::ServerComposition::with_registry(registry); + if let Some(backend_tls) = backend_tls { + composition = composition.add_pipeline_validator(move |ctx| { + backend_tls_unchanged(ctx.config(), &backend_tls) + .map_err(|error| praxis::CompositionError::new(error.to_string())) + }); + } + let result = praxis::try_run_server_with_composition(config, composition, config_file, log_level); drop(grid_runtime); let exit_code = result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS); // The Praxis guard shuts down the OTLP provider and flushes queued spans. @@ -213,11 +233,11 @@ fn start_metrics_listener(config: &Config) -> Result<(), String> { /// registering the filters. fn start_grid_routing( path: &str, - praxis_config: &Config, + backend_tls: BTreeMap, registry: &mut praxis_filter::FilterRegistry, ) -> Result { let config = ai_grid_filters::load_serving_config(path)?; - let mut runtime = ai_grid_filters::spawn_grid_routing(&config)?; + let mut runtime = ai_grid_filters::spawn_grid_routing(&config, backend_tls)?; ai_grid_filters::register_grid_filters( registry, runtime.snapshot(), @@ -252,7 +272,7 @@ struct BackendCluster { } /// TLS properties required for an authenticated provider hop. -#[derive(Deserialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] struct BackendTls { /// Expected server name. sni: String, @@ -265,14 +285,14 @@ struct BackendTls { } /// CA trust input used by the load balancer. -#[derive(Deserialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] struct BackendCa { /// CA certificate path. ca_path: String, } /// Client identity used by the load balancer. -#[derive(Deserialize)] +#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] struct BackendClientCert { /// Client certificate path. cert_path: String, @@ -280,19 +300,12 @@ struct BackendClientCert { key_path: String, } -/// Only verified mutual-TLS backends in Grid-enabled chains can carry hop context. +/// Resolve unique verified mutual-TLS backends across all chains. #[expect(clippy::too_many_lines, reason = "keeps the backend transport checks together")] -fn provider_hop_backends(config: &Config) -> Result, praxis_filter::FilterError> { +fn provider_hop_backends(config: &Config) -> Result, praxis_filter::FilterError> { let mut backends = BTreeMap::new(); let mut unverified = BTreeSet::new(); for chain in &config.filter_chains { - if !chain - .filters - .iter() - .any(|filter| filter.filter_type == "grid_site_route") - { - continue; - } for filter in chain .filters .iter() @@ -313,7 +326,7 @@ fn provider_hop_backends(config: &Config) -> Result, pr .is_some_and(|cert| !cert.cert_path.trim().is_empty() && !cert.key_path.trim().is_empty()) }); if let Some(tls) = verified_sni { - if unverified.contains(&backend.name) || backends.insert(backend.name.clone(), tls.sni).is_some() { + if unverified.contains(&backend.name) || backends.insert(backend.name.clone(), tls).is_some() { return Err(format!("grid: ambiguous provider-hop backend {:?}", backend.name).into()); } } else { @@ -328,6 +341,22 @@ fn provider_hop_backends(config: &Config) -> Result, pr Ok(backends) } +/// Reject a Praxis config reload that changes a backend identity trusted by the Grid runtime. +fn backend_tls_unchanged( + config: &Config, + expected: &BTreeMap, +) -> Result<(), praxis_filter::FilterError> { + let current = provider_hop_backends(config)?; + for (cluster, tls) in expected { + if current.get(cluster) != Some(tls) { + return Err( + format!("grid: backend {cluster:?} changed its verified TLS identity; restart Grid routing").into(), + ); + } + } + Ok(()) +} + /// Usage line for a malformed command line. const USAGE: &str = "usage: grid-gateway [--config | -c | ]"; @@ -359,11 +388,22 @@ fn config_arg>(args: I) -> Result, mod tests { use std::ffi::OsStr; - use super::{USAGE, config_arg, otlp_headers_present, validate_otlp_endpoint_transport_values}; + use super::{ + USAGE, backend_tls_unchanged, config_arg, otlp_headers_present, provider_hop_backends, + validate_otlp_endpoint_transport_values, + }; + + fn config_with_backend(backend: &str) -> praxis_core::config::Config { + let backend = format!(" {}", backend.replace('\n', "\n ")); + let yaml = format!( + "listeners:\n - name: test\n address: 127.0.0.1:8080\n filter_chains: [grid]\nfilter_chains:\n - name: grid\n filters:\n - filter: grid_site_route\n - filter: load_balancer\n clusters:\n - name: provider-a\n{backend}" + ); + praxis_core::config::Config::from_yaml(&yaml).expect("backend fixture") + } #[test] fn provider_hop_backends_reject_verified_plaintext_name_collision() { - let verified = " tls:\n sni: provider-a.grid.internal\n verify: true\n ca: { ca_path: /tls/ca.crt }\n client_cert: { cert_path: /tls/tls.crt, key_path: /tls/tls.key }\n"; + let verified = " tls:\n sni: provider-a.grid.internal\n verify: true\n ca: { ca_path: /tls/ca.crt }\n client_cert: { cert_path: /tls/tls.crt, key_path: /tls/tls.key }\n endpoints: [provider-a:8443]\n"; let plaintext = " endpoints: [provider-a:80]\n"; for (first, second) in [(verified, plaintext), (plaintext, verified)] { let mut config = config_with_backend(first); @@ -378,6 +418,29 @@ mod tests { } } + #[test] + fn backend_reload_keeps_the_original_verified_provider_hop_identity() { + let verified = " tls:\n sni: provider-a.grid.internal\n verify: true\n ca: { ca_path: /tls/ca.crt }\n client_cert: { cert_path: /tls/tls.crt, key_path: /tls/tls.key }\n endpoints: [provider-a:8443]\n"; + let plaintext = " endpoints: [provider-a:80]\n"; + let expected = provider_hop_backends(&config_with_backend(verified)).expect("verified backend"); + let changed_sni = verified.replace("provider-a.grid.internal", "other.grid.internal"); + let changed_ca = verified.replace("/tls/ca.crt", "/tls/other-ca.crt"); + for (label, backend) in [ + ("plaintext", plaintext), + ("changed SNI", changed_sni.as_str()), + ("changed CA", changed_ca.as_str()), + ] { + assert!( + backend_tls_unchanged(&config_with_backend(backend), &expected).is_err(), + "{label} reload cannot inherit provider-hop trust" + ); + assert!( + backend_tls_unchanged(&config_with_backend(verified), &expected).is_ok(), + "rejection must leave the original identity trusted" + ); + } + } + fn parse(args: &[&str]) -> Result, String> { config_arg(args.iter().map(|arg| (*arg).to_owned())) } diff --git a/gateway/tests/serving_reload.rs b/gateway/tests/serving_reload.rs index 075055843..539f11d72 100644 --- a/gateway/tests/serving_reload.rs +++ b/gateway/tests/serving_reload.rs @@ -18,7 +18,7 @@ mod tests { process::{Child, Command, Stdio}, sync::{ Arc, Mutex, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, }, thread, time::{Duration, Instant}, @@ -80,6 +80,7 @@ mod tests { struct Peer { addr: String, seen: Arc>>, + hop_requests: Arc, stop: Arc, } @@ -108,6 +109,7 @@ mod tests { let listener = TcpListener::bind("127.0.0.1:0").expect("bind peer"); let addr = listener.local_addr().expect("peer addr").to_string(); let seen = Arc::new(Mutex::new(Vec::new())); + let hop_requests = Arc::new(AtomicUsize::new(0)); let stop = Arc::new(AtomicBool::new(false)); let body = format!( "HTTP/1.1 200 OK\r\nDate: Thu, 01 Jan 1970 00:00:01 GMT\r\nContent-Length: {len}\r\nConnection: close\r\n\r\n{line}", @@ -115,6 +117,7 @@ mod tests { line = signals_line(site), ); let (recorded, stopping) = (Arc::clone(&seen), Arc::clone(&stop)); + let recorded_hops = Arc::clone(&hop_requests); thread::spawn(move || { for stream in listener.incoming() { if stopping.load(Ordering::SeqCst) { @@ -134,6 +137,12 @@ mod tests { Ok(n) => request.extend_from_slice(&buf[..n]), } } + if String::from_utf8_lossy(&request) + .to_ascii_lowercase() + .contains("x-ai-routing-candidate: candidate-east") + { + recorded_hops.fetch_add(1, Ordering::SeqCst); + } let Some(leaf) = tls.conn.peer_certificates().and_then(<[_]>::first) else { continue; }; @@ -141,7 +150,12 @@ mod tests { let _sent = tls.write_all(body.as_bytes()).and_then(|()| tls.flush()); } }); - Self { addr, seen, stop } + Self { + addr, + seen, + hop_requests, + stop, + } } fn polls(&self) -> usize { @@ -151,6 +165,10 @@ mod tests { fn saw(&self, fingerprint: &[u8; 32]) -> bool { self.seen.lock().expect("seen").contains(fingerprint) } + + fn hops(&self) -> usize { + self.hop_requests.load(Ordering::SeqCst) + } } impl Drop for Peer { @@ -223,20 +241,22 @@ mod tests { struct Gateway { child: Child, output: Arc>>, + listen: u16, admin: u16, } impl Gateway { fn start(work: &Path, serving: &Mount) -> Self { let (listen, admin) = (free_port(), free_port()); + let config = format!( + "admin:\n address: \"127.0.0.1:{admin}\"\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters:\n - filter: static_response\n status: 200\n" + ); + Self::start_with_config(work, serving, listen, admin, &config) + } + + fn start_with_config(work: &Path, serving: &Mount, listen: u16, admin: u16, yaml: &str) -> Self { let config = work.join("praxis.yaml"); - std::fs::write( - &config, - format!( - "admin:\n address: \"127.0.0.1:{admin}\"\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters:\n - filter: static_response\n status: 200\n" - ), - ) - .expect("praxis config"); + std::fs::write(&config, yaml).expect("praxis config"); let mut child = Command::new(env!("CARGO_BIN_EXE_grid-gateway")) .arg("--config") .arg(&config) @@ -249,7 +269,27 @@ mod tests { let output = Arc::new(Mutex::new(Vec::new())); drain(child.stdout.take().expect("stdout"), Arc::clone(&output)); drain(child.stderr.take().expect("stderr"), Arc::clone(&output)); - Self { child, output, admin } + Self { + child, + output, + listen, + admin, + } + } + + fn route_status(&self) -> u16 { + let mut stream = TcpStream::connect(("127.0.0.1", self.listen)).expect("connect gateway"); + stream + .write_all(b"GET /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\nX-Model: llama\r\nConnection: close\r\n\r\n") + .expect("send request"); + let mut response = String::new(); + stream.read_to_string(&mut response).expect("read response"); + response + .split_whitespace() + .nth(1) + .expect("HTTP status") + .parse() + .expect("numeric HTTP status") } /// The rejected count from the admin metrics, zero until the series exists. @@ -354,4 +394,84 @@ mod tests { drop(gateway); let _cleaned = std::fs::remove_dir_all(&work); } + + #[test] + fn praxis_backend_reload_cannot_inherit_provider_hop_trust() { + let work = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("backend-reload-{}", std::process::id())); + let _cleared = std::fs::remove_dir_all(&work); + let ca = generate_ca("grid-ca").expect("ca"); + let local = generate_site_cert(&ca, "local").expect("client cert"); + let mut identity = Mount::new(work.join("identity")); + identity.write(&[ + ("ca.crt", ca.cert_pem.as_bytes()), + ("tls.crt", local.cert_pem.as_bytes()), + ("tls.key", local.key_pem.as_bytes()), + ]); + let backend = Peer::start(&ca, "east"); + let mut serving = Mount::new(work.join("serving")); + serving.write(&[( + "serving-config.json", + br#"{"local_site":"local","window_secs":60,"load_window_ms":30000,"candidates":[{"kind":"inference_model","name":"llama","site":"local","cluster":"pool-east","stable_id":"candidate-east"}],"provider_hop_clusters":["pool-east"],"provider_hop_sni":{"pool-east":"east.grid.internal"},"peers":[]}"#, + )]); + let (listen, admin) = (free_port(), free_port()); + let config = |tls: &str| { + format!( + "insecure_options:\n allow_private_endpoints: true\nadmin:\n address: \"127.0.0.1:{admin}\"\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters:\n - filter: grid_site_route\n model_header: X-Model\n - filter: load_balancer\n clusters:\n - name: pool-east\n{tls} endpoints: [\"{addr}\"]\n", + addr = backend.addr + ) + }; + let verified = format!( + " tls:\n ca: {{ ca_path: {ca} }}\n client_cert: {{ cert_path: {cert}, key_path: {key} }}\n sni: east.grid.internal\n verify: true\n", + ca = identity.path("ca.crt"), + cert = identity.path("tls.crt"), + key = identity.path("tls.key"), + ); + let gateway = Gateway::start_with_config(&work, &serving, listen, admin, &config(&verified)); + gateway.eventually("verified backend route", || { + TcpStream::connect(("127.0.0.1", listen)).is_ok() + }); + assert_eq!(gateway.route_status(), 200, "the original verified backend routes"); + assert_eq!(backend.hops(), 1, "the verified route carries its candidate identity"); + + for (label, tls) in [ + ("plaintext", String::new()), + ( + "changed SNI", + verified.replace("east.grid.internal", "other.grid.internal"), + ), + ] { + let before = gateway + .output + .lock() + .expect("output") + .iter() + .filter(|line| line.contains("changed its verified TLS identity")) + .count(); + std::fs::write(work.join("praxis.yaml"), config(&tls)).expect("rewrite praxis config"); + gateway.eventually(label, || { + gateway + .output + .lock() + .expect("output") + .iter() + .filter(|line| line.contains("changed its verified TLS identity")) + .count() + > before + }); + let previous_hops = backend.hops(); + assert_eq!( + gateway.route_status(), + 200, + "{label} reload leaves the verified route live" + ); + assert_eq!( + backend.hops(), + previous_hops + 1, + "{label} reload keeps hop context on the original verified route" + ); + } + + drop(gateway); + let _cleaned = std::fs::remove_dir_all(&work); + } } diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 562b42734..562918f93 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -2242,6 +2242,26 @@ async fn reconcile_routing_overlay_inner( "routing overlay has no candidates; distributing authoritative no-route state" ); } + let credential_bearing = overlay + .candidates + .iter() + .any(|candidate| candidate.credential.is_some()); + if credential_bearing + && let Some(cc) = gw_ref.consumer_config.as_ref().filter(|cc| cc.enabled) + && !cc.enable_projected_credentials + { + let error = OperatorError::ConsumerConfigRender(ConsumerConfigError::ProjectedCredentialsUnsupported); + consumer_statuses.push(consumer_config_status_error(gw_ref, cc, &error, observed_generation)); + overlay_statuses.push(retained_overlay_status( + network, + gw_ref, + observed_generation, + Some(&render), + "ProjectedCredentialsUnsupported", + "credential-bearing overlay requires a rolled-out projected credential filter and mounted Secrets", + )); + continue; + } // For delegated gateways, make the generated config and all of its // Secret mounts available before publishing candidates that can use // them. A pending or failed mount/config rollout retains the previous @@ -2266,7 +2286,7 @@ async fn reconcile_routing_overlay_inner( .await { Ok(outcome) => { - if outcome.config_applied { + if outcome.config_applied && (!credential_bearing || cc.supports_projected_credentials) { consumer_statuses.push(consumer_config_status_rendered(gw_ref, cc, observed_generation)); } if let Some(status) = outcome.mount_status { @@ -2305,6 +2325,23 @@ async fn reconcile_routing_overlay_inner( } } + if credential_bearing + && let Some(cc) = gw_ref.consumer_config.as_ref().filter(|cc| cc.enabled) + && !cc.supports_projected_credentials + { + let error = OperatorError::ConsumerConfigRender(ConsumerConfigError::ProjectedCredentialsUnsupported); + consumer_statuses.push(consumer_config_status_error(gw_ref, cc, &error, observed_generation)); + overlay_statuses.push(retained_overlay_status( + network, + gw_ref, + observed_generation, + Some(&render), + "ProjectedCredentialsUnsupported", + "credential-bearing overlay requires a rolled-out projected credential filter and mounted Secrets", + )); + continue; + } + let resource_version = match distribute_overlay_configmap(&overlay, &render, network_name, gw_ref, client).await { Ok(rv) => rv, @@ -2406,10 +2443,6 @@ async fn reconcile_routing_overlay_inner( }) } -/// Legacy status reason used by retained-status compatibility tests. -#[cfg(test)] -const EMPTY_CANDIDATES: &str = "EmptyCandidates"; - /// What one routing overlay pass produced, per gateway. struct OverlayOutcome { /// Consumer config render and apply results. @@ -2485,8 +2518,10 @@ fn render_serving_text( cc.tls_cert_mount_path.as_str() }); let provider_hop_sni = serving_provider_hop_sni_for_overlay(overlay, gw_ref)?; - let provider_hop_clusters = provider_hop_sni.keys().cloned().collect(); + let provider_hop_clusters: BTreeSet = provider_hop_sni.keys().cloned().collect(); let inputs = ServingInputs { + provider_hop_clusters: &provider_hop_clusters, + provider_hop_sni: &provider_hop_sni, tls_mount, local_signals_addr: source.settings.local_signals_addr.as_deref(), pins: &source.pins, @@ -2500,13 +2535,13 @@ fn render_serving_text( fn serving_provider_hop_sni_for_overlay( overlay: &routing_overlay::RoutingOverlay, gw_ref: &GatewayRef, -) -> Result, OperatorError> { +) -> Result, OperatorError> { if overlay.candidates.is_empty() { if let Err(error) = serving_provider_hop_clusters(gw_ref) { tracing::warn!(gateway = %gw_ref.name, %error, "invalid provider-hop declaration ignored for empty serving revision"); } - Ok(std::collections::BTreeMap::new()) + Ok(BTreeMap::new()) } else { serving_provider_hop_clusters(gw_ref)?; Ok(gw_ref @@ -2889,7 +2924,7 @@ async fn apply_consumer_config_for_gateway( observed_generation: i64, client: &Client, ) -> Result { - let rendered = consumer_config::render_consumer_config( + let rendered = consumer_config::render_consumer_config_with_projected( overlay, &cc.credential_mount_base, &cc.cluster_endpoints, @@ -2899,6 +2934,7 @@ async fn apply_consumer_config_for_gateway( &gw_ref.name, &gw_ref.namespace, cc.telemetry.as_ref(), + cc.enable_projected_credentials, cc.mount_reconciliation.as_ref().is_some_and(|mounts| mounts.enabled), )?; if let Some(mounts) = cc.mount_reconciliation.as_ref().filter(|mounts| mounts.enabled) { @@ -4463,7 +4499,10 @@ fn provider_state_from_kube( let provider_id = provider.metadata.name.as_deref()?; let routing_cluster = routing_overlay::routing_identity(provider)?.to_owned(); let models = provider.spec.models.iter().map(|m| m.name.clone()).collect(); - let phase = crdt_phase_from_provider(provider.status.as_ref().map(|s| &s.phase)); + let phase = crdt_phase_from_provider( + provider.status.as_ref(), + provider.metadata.generation.unwrap_or_default(), + ); let revision = provider_revision(&provider.metadata); let capacity_weight = effective_capacity_weight(provider); Some(crdt::ProviderState { @@ -4516,7 +4555,15 @@ fn tool_provider_state_from_kube( ) -> Option { let name = provider.metadata.name.as_deref()?; let tools = tool_names_from_agent_tool_provider(provider); - let phase = crdt_phase_from_provider(provider.status.as_ref().map(|s| &s.phase)); + let phase = provider + .status + .as_ref() + .map_or(crdt::ProviderPhase::Pending, |status| match status.phase { + crate::crd::inference_provider::ProviderPhase::Available => crdt::ProviderPhase::Available, + crate::crd::inference_provider::ProviderPhase::Degraded => crdt::ProviderPhase::Degraded, + crate::crd::inference_provider::ProviderPhase::Unavailable => crdt::ProviderPhase::Unavailable, + crate::crd::inference_provider::ProviderPhase::Pending => crdt::ProviderPhase::Pending, + }); let revision = provider_revision(&provider.metadata); // Prefix with "tool/" to distinguish from InferenceProvider names in the // CRDT key (network/site/provider_id). Without this, an InferenceProvider @@ -5166,6 +5213,9 @@ pub(crate) fn consumer_config_status_error( OperatorError::ConsumerConfigRender(ConsumerConfigError::MissingSni { .. }) => "MissingSni", OperatorError::ConsumerConfigRender(ConsumerConfigError::PlaintextWithSni { .. }) => "PlaintextWithSni", OperatorError::ConsumerConfigRender(ConsumerConfigError::NoInferenceCandidates) => "NoInferenceCandidates", + OperatorError::ConsumerConfigRender(ConsumerConfigError::ProjectedCredentialsUnsupported) => { + "ProjectedCredentialsUnsupported" + }, OperatorError::ConsumerConfigRender(_) => "ConsumerConfigRenderFailed", OperatorError::MountReconciliation(failure) => failure.reason, OperatorError::Kube(_) => "ConsumerConfigApplyFailed", @@ -6050,10 +6100,7 @@ fn parse_metrics_refresh_interval(value: &str) -> Result Result, ) -> Result { + if overlay.local_site.trim().is_empty() { + return Err(ConsumerConfigError::BlankLocalSite); + } + if cluster_endpoints.is_empty() { + return Err(ConsumerConfigError::NoClusterEndpoints); + } + let hop_clusters = render_provider_hop_clusters(cluster_endpoints)?; + let endpoint_map: BTreeMap<&str, &ClusterEndpointConfig> = + cluster_endpoints.iter().map(|ep| (ep.cluster.as_str(), ep)).collect(); + for candidate in overlay + .candidates + .iter() + .filter(|candidate| candidate.kind == INFERENCE_MODEL) + { + if !endpoint_map.contains_key(candidate.cluster.as_str()) { + return Err(ConsumerConfigError::MissingClusterEndpoint { + cluster: candidate.cluster.clone(), + }); + } + } + if let Some(telemetry) = telemetry { + telemetry.validate().map_err(ConsumerConfigError::InvalidTelemetry)?; + } + let mut endpoints = cluster_endpoints.iter().collect::>(); + endpoints.sort_by(|left, right| left.cluster.cmp(&right.cluster)); + let clusters = endpoints + .into_iter() + .map(|endpoint| { + let quoted = yaml_scalar(&endpoint.cluster)?; + render_cluster_entry("ed, endpoint, tls_cert_mount_path) + }) + .collect::, ConsumerConfigError>>()? + .join("\n"); + let network = yaml_scalar(&overlay.network)?; + let gateway = yaml_scalar(gateway_name)?; + let namespace = yaml_scalar(gateway_namespace)?; + let local_site = yaml_scalar(&overlay.local_site)?; + let trace_context_filter = if telemetry.is_some() { + " - filter: trace_context\n" + } else { + "" + }; let mut config = format!( - "listeners:\n\ - \x20 - name: public\n\ - \x20 address: \"0.0.0.0:{listener_port}\"\n\ - \x20 filter_chains: [no-provider-chain]\n\ - filter_chains:\n\ - \x20 - name: no-provider-chain\n\ - \x20 filters:\n\ - \x20 - filter: static_response\n\ - \x20 status: 503\n\ - \x20 headers:\n\ - \x20 - name: Content-Type\n\ - \x20 value: application/json\n\ - \x20 body: '{{\"error\":\"no providers available\"}}'\n" + concat!( + "listeners:\n", + " - name: public\n", + " address: \"0.0.0.0:{listener_port}\"\n", + " filter_chains: [consumer-chain]\n", + "filter_chains:\n", + " - name: consumer-chain\n", + " filters:\n", + "{trace_context_filter}", + " - filter: json_body_field\n", + " field: model\n", + " header: X-Model\n", + " - filter: intelligent_route\n", + " local_site: {local_site}\n", + " model_header: X-Model\n", + " overlay_file: {overlay_file}\n", + " expected_overlay_scope:\n", + " network: {network}\n", + " gateway: {gateway}\n", + " namespace: {namespace}\n", + " local_site: {local_site}\n", + " reload:\n", + " enabled: true\n", + "{hop_clusters}", + ), + listener_port = listener_port, + trace_context_filter = trace_context_filter, + local_site = local_site, + overlay_file = CONSUMER_OVERLAY_FILE, + network = network, + gateway = gateway, + namespace = namespace, + hop_clusters = hop_clusters, ); + let candidates: Vec<_> = overlay + .candidates + .iter() + .filter(|candidate| candidate.kind == INFERENCE_MODEL) + .collect(); + let inject = if projected_credentials { + render_credential_inject(&[], credential_mount_base, &overlay.local_site, true) + } else { + render_credential_inject(&candidates, credential_mount_base, &overlay.local_site, false) + }; + config.push_str(&inject); + config.push_str("\n - filter: load_balancer\n clusters:\n"); + config.push_str(&clusters); if let Some(telemetry) = telemetry { config.push_str(&render_telemetry(telemetry)?); } - config.push_str("admin:\n address: \"127.0.0.1:9901\"\nshutdown_timeout_secs: 5\n"); + config.push_str("\nadmin:\n address: \"127.0.0.1:9901\"\nshutdown_timeout_secs: 5\n"); Ok(config) } +/// Test-facing wrapper for the production gateway config renderer. +#[cfg(test)] +#[expect(clippy::too_many_arguments, reason = "matches the gateway renderer inputs")] +fn generate_consumer_praxis_config_for_gateway( + overlay: &RoutingOverlay, + credential_mount_base: &str, + cluster_endpoints: &[ClusterEndpointConfig], + tls_cert_mount_path: &str, + listener_port: u16, + gateway_name: &str, + gateway_namespace: &str, + projected_credentials: bool, +) -> Result { + generate_consumer_praxis_config_for_gateway_with_telemetry( + overlay, + credential_mount_base, + cluster_endpoints, + tls_cert_mount_path, + listener_port, + gateway_name, + gateway_namespace, + projected_credentials, + None, + ) +} + /// Render Praxis YAML and its Secret-file requirements from the same inputs. /// /// Only credential-bearing candidates whose site is this gateway's local site @@ -491,7 +623,7 @@ fn render_no_provider_praxis_config( clippy::too_many_lines, reason = "configuration and mount requirements are derived from one candidate pass" )] -pub(crate) fn render_consumer_config( +pub(crate) fn render_consumer_config_with_projected( overlay: &RoutingOverlay, credential_mount_base: &str, cluster_endpoints: &[ClusterEndpointConfig], @@ -501,6 +633,7 @@ pub(crate) fn render_consumer_config( gateway_name: &str, gateway_namespace: &str, telemetry: Option<&GatewayTelemetryConfig>, + projected_credentials: bool, delegated_mounts: bool, ) -> Result { let inference_candidates: Vec<&RoutingCandidate> = overlay @@ -508,18 +641,17 @@ pub(crate) fn render_consumer_config( .iter() .filter(|candidate| candidate.kind == INFERENCE_MODEL) .collect(); - let config_yaml = if inference_candidates.is_empty() { - render_no_provider_praxis_config(listener_port, telemetry)? - } else { - generate_consumer_praxis_config_with_telemetry( - overlay, - credential_mount_base, - cluster_endpoints, - tls_cert_mount_path, - listener_port, - telemetry, - )? - }; + let config_yaml = generate_consumer_praxis_config_for_gateway_with_telemetry( + overlay, + credential_mount_base, + cluster_endpoints, + tls_cert_mount_path, + listener_port, + gateway_name, + gateway_namespace, + projected_credentials, + telemetry, + )?; let mut requirements = BTreeMap::<(MountPurpose, String, String, String), BTreeSet>::new(); for candidate in &inference_candidates { @@ -529,11 +661,18 @@ pub(crate) fn render_consumer_config( let Some(credential) = candidate.credential.as_ref() else { continue; }; - let path = credential_file_path( - credential_mount_base, - &credential.secret_ref.name, - &credential.secret_ref.key, - ); + let path = if projected_credentials { + format!( + "{credential_mount_base}/{}/{}/{}", + credential.secret_ref.namespace, credential.secret_ref.name, credential.secret_ref.key + ) + } else { + credential_file_path( + credential_mount_base, + &credential.secret_ref.name, + &credential.secret_ref.key, + ) + }; add_requirement( &mut requirements, MountPurpose::BackendCredential, @@ -545,9 +684,9 @@ pub(crate) fn render_consumer_config( )?; } - let used_clusters: BTreeSet<&str> = inference_candidates + let used_clusters: BTreeSet<&str> = cluster_endpoints .iter() - .map(|candidate| candidate.cluster.as_str()) + .map(|endpoint| endpoint.cluster.as_str()) .collect(); let needs_mutual_tls = cluster_endpoints.iter().any(|endpoint| { used_clusters.contains(endpoint.cluster.as_str()) @@ -632,6 +771,36 @@ pub(crate) fn render_consumer_config( }) } +/// Exercise the default renderer contract in local tests. +#[cfg(test)] +#[expect(clippy::too_many_arguments, reason = "matches the production renderer inputs")] +fn render_consumer_config( + overlay: &RoutingOverlay, + credential_mount_base: &str, + cluster_endpoints: &[ClusterEndpointConfig], + tls_cert_mount_path: &str, + listener_port: u16, + tls: &TlsConfig, + gateway_name: &str, + gateway_namespace: &str, + telemetry: Option<&GatewayTelemetryConfig>, + delegated_mounts: bool, +) -> Result { + render_consumer_config_with_projected( + overlay, + credential_mount_base, + cluster_endpoints, + tls_cert_mount_path, + listener_port, + tls, + gateway_name, + gateway_namespace, + telemetry, + false, + delegated_mounts, + ) +} + /// Add one validated file reference to the grouped requirements map. #[expect( clippy::type_complexity, @@ -773,6 +942,7 @@ pub(crate) fn build_consumer_config_map( /// /// Each candidate is indented and includes `credential.secretRef` when present. /// Token values are never included. +#[cfg(test)] fn render_candidates(candidates: &[&RoutingCandidate], local_site: &str) -> String { candidates .iter() @@ -782,6 +952,7 @@ fn render_candidates(candidates: &[&RoutingCandidate], local_site: &str) -> Stri } /// Render the explicit Grid-owned request-selection policy. +#[cfg(test)] fn render_selection_policy(policy: Option<&crate::crd::grid_network::SelectionPolicyConfig>) -> String { let Some(policy) = policy else { return String::new(); @@ -867,6 +1038,7 @@ pub(crate) fn provider_hop_clusters( clippy::too_many_lines, reason = "Candidate YAML fields are kept together to mirror the wire contract." )] +#[cfg(test)] fn render_candidate(c: &RoutingCandidate, include_credential: bool) -> String { let mut lines = vec![ format!( @@ -906,6 +1078,7 @@ fn render_candidate(c: &RoutingCandidate, include_credential: bool) -> String { } /// Render the `credential.secretRef` block for one candidate. +#[cfg(test)] fn render_credential_reference(cred: &crate::resources::routing_overlay::ProjectedCredential) -> Vec { vec![ " credential:".to_owned(), @@ -941,7 +1114,8 @@ fn render_credential_inject( candidates: &[&RoutingCandidate], credential_mount_base: &str, local_site: &str, -) -> Option { + enable_projected_credentials: bool, +) -> String { // Collect unique (strategy, name, namespace, key) → rendered entry. // BTreeMap provides deterministic sorted order by key. let mut entries: BTreeMap<(String, String, String, String), String> = BTreeMap::new(); @@ -1012,6 +1186,7 @@ fn render_credential_inject( /// deterministically. Every cluster must have a matching entry in /// `cluster_endpoints` with explicit transport configuration; missing /// endpoint, missing transport, or missing SNI on mTLS all fail closed. +#[cfg(test)] fn render_load_balancer( candidates: &[&RoutingCandidate], cluster_endpoints: &[ClusterEndpointConfig], @@ -1353,9 +1528,9 @@ mod tests { } #[test] - fn empty_candidate_config_fails_closed_without_an_invalid_route_filter() { + fn empty_candidate_config_requires_endpoint_inventory_for_restoration() { let overlay = simple_overlay(Vec::new()); - let rendered = render_consumer_config( + let error = render_consumer_config( &overlay, MOUNT_BASE, &[], @@ -1367,22 +1542,11 @@ mod tests { None, false, ) - .unwrap_or_else(|_| std::process::abort()); - let parsed: serde_yaml::Value = - serde_yaml::from_str(&rendered.config_yaml).unwrap_or_else(|_| std::process::abort()); - let filters = parsed - .get("filter_chains") - .and_then(serde_yaml::Value::as_sequence) - .and_then(|chains| chains.first()) - .and_then(|chain| chain.get("filters")) - .and_then(serde_yaml::Value::as_sequence) - .unwrap_or_else(|| std::process::abort()); - - assert_eq!(rendered.requirements.len(), 0); - assert_eq!(filters[0]["filter"], "static_response"); - assert_eq!(filters[0]["status"], 503); - assert!(!rendered.config_yaml.contains("intelligent_route")); - assert!(rendered.config_yaml.contains("no providers available")); + .expect_err("restoration needs an endpoint inventory"); + assert!( + matches!(error, ConsumerConfigError::NoClusterEndpoints), + "missing inventory must fail closed" + ); } #[test] @@ -1656,6 +1820,7 @@ mod tests { endpoints[0].transport = Some(EndpointTransport { mode: TransportMode::MutualTls, sni: Some("provider-a.grid.internal".to_owned()), + ca_secret_ref: None, }); endpoints.push(ClusterEndpointConfig { cluster: "provider-b".to_owned(), @@ -1663,6 +1828,7 @@ mod tests { transport: Some(EndpointTransport { mode: TransportMode::Plaintext, sni: None, + ca_secret_ref: None, }), }); @@ -1723,6 +1889,7 @@ mod tests { transport: Some(EndpointTransport { mode: TransportMode::Plaintext, sni: None, + ca_secret_ref: None, }), }]; let yaml = generate_consumer_praxis_config_for_gateway( @@ -1766,6 +1933,7 @@ mod tests { transport: Some(EndpointTransport { mode: TransportMode::Plaintext, sni: None, + ca_secret_ref: None, }), }]; let yaml = generate_consumer_praxis_config_for_gateway( @@ -2806,6 +2974,7 @@ mod tests { } #[test] + #[expect(clippy::too_many_lines, reason = "checks every rendered mTLS property")] fn cluster_with_mtls_transport_renders_mtls_entry() { let endpoints = [mtls_ep("site-a", "172.18.0.4:30080", "site-a.grid.internal")]; let overlay = simple_overlay(vec![plain_candidate( diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index d8f55dd07..f1c94753a 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -86,6 +86,9 @@ pub(crate) struct ServingConfig { /// One routable `(model, site, cluster)`, `cluster` naming a `load_balancer` cluster. #[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub(crate) struct ServingCandidate { + /// Overlay-assigned stable identity used by authenticated provider hops. + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) stable_id: Option, /// Always `inference_model`. pub(crate) kind: &'static str, /// Model name. @@ -141,6 +144,10 @@ pub(crate) struct ServingPeer { /// Per-gateway inputs that are not in the overlay. #[derive(Clone, Debug)] pub(crate) struct ServingInputs<'input> { + /// Verified gateway clusters allowed to carry provider-hop context. + pub(crate) provider_hop_clusters: &'input BTreeSet, + /// Declared TLS identity for each provider-hop cluster. + pub(crate) provider_hop_sni: &'input BTreeMap, /// Directory holding this site's `ca.crt`, `tls.crt`, `tls.key` in the gateway pod. pub(crate) tls_mount: &'input str, /// Operator-configured address of this site's own signals endpoint. @@ -165,7 +172,7 @@ where { let candidates = candidates(overlay); let sites: BTreeSet<&str> = candidates.iter().map(|candidate| candidate.site.as_str()).collect(); - let mut addrs = remote_addrs(members, &sites, &overlay.local_site); + let mut addrs = remote_addrs(members, &sites, &overlay.local_site, overlay.candidates.is_empty()); if let Some(addr) = inputs.local_signals_addr && certs::validate_site_name(&overlay.local_site).is_ok() { @@ -225,7 +232,7 @@ type CandidateKey<'overlay> = (bool, &'overlay str, &'overlay str, &'overlay str fn dedup<'overlay>( overlay: impl Iterator, local_site: &str, -) -> BTreeMap, (bool, AdmissionState)> { +) -> BTreeMap, (bool, AdmissionState, Option)> { let mut unique = BTreeMap::new(); for candidate in overlay.filter(|candidate| routable(candidate)) { let key = ( @@ -238,9 +245,14 @@ fn dedup<'overlay>( // Seeded with the identities of both folds, so the first candidate and every // duplicate after it take the same path: fresh only if all are, admission as // restricted as the most restricted. - let (fresh, held) = unique.entry(key).or_insert((true, AdmissionState::NewAndExisting)); + let (fresh, held, stable_id) = unique + .entry(key) + .or_insert_with(|| (true, AdmissionState::NewAndExisting, candidate.stable_id.clone())); *fresh &= candidate.fresh; *held = (*held).max(admission); + if stable_id.is_none() { + stable_id.clone_from(&candidate.stable_id); + } } unique } @@ -249,6 +261,9 @@ fn dedup<'overlay>( /// /// Past the cap, candidates taking new requests are kept before the rest. fn candidates(overlay: &RoutingOverlay) -> Vec { + if overlay.candidates.is_empty() { + return Vec::new(); + } // Local first so cold start, before any signal, prefers this site. let unique = dedup(overlay.candidates.iter().chain(&overlay.excluded), &overlay.local_site); let mut kept: Vec<_> = unique.into_iter().collect(); @@ -257,19 +272,22 @@ fn candidates(overlay: &RoutingOverlay) -> Vec { candidates = kept.len(), "serving config: dropping candidates past the gateway cap" ); - kept.sort_by_key(|(key, (_, admission))| (*admission != AdmissionState::NewAndExisting, *key)); + kept.sort_by_key(|(key, (_, admission, _))| (*admission != AdmissionState::NewAndExisting, *key)); kept.truncate(MAX_CANDIDATES); kept.sort_by_key(|(key, _)| *key); } kept.into_iter() - .map(|((_, site, name, cluster), (fresh, admission))| ServingCandidate { - kind: INFERENCE_MODEL, - name: name.to_owned(), - site: site.to_owned(), - cluster: cluster.to_owned(), - fresh, - admission, - }) + .map( + |((_, site, name, cluster), (fresh, admission, stable_id))| ServingCandidate { + stable_id, + kind: INFERENCE_MODEL, + name: name.to_owned(), + site: site.to_owned(), + cluster: cluster.to_owned(), + fresh, + admission, + }, + ) .collect() } @@ -323,13 +341,16 @@ fn remote_addrs<'member, Members>( members: Members, sites: &BTreeSet<&str>, local_site: &str, + include_all: bool, ) -> BTreeMap where Members: IntoIterator, { members .into_iter() - .filter(|(site, _)| *site != local_site && sites.contains(site) && certs::validate_site_name(site).is_ok()) + .filter(|(site, _)| { + *site != local_site && (include_all || sites.contains(site)) && certs::validate_site_name(site).is_ok() + }) .filter_map(|(site, endpoint)| { let Some(parsed) = SignalsEndpoint::parse(endpoint).filter(SignalsEndpoint::is_dialable) else { tracing::warn!(site, endpoint, "serving config: refusing advertised peer address"); @@ -474,6 +495,8 @@ mod tests { static NO_PROVIDER_HOP_SNI: BTreeMap = BTreeMap::new(); const INPUTS: ServingInputs<'static> = ServingInputs { + provider_hop_clusters: &NO_PROVIDER_HOPS, + provider_hop_sni: &NO_PROVIDER_HOP_SNI, tls_mount: "/etc/praxis/tls", local_signals_addr: None, pins: &NO_PINS, @@ -541,9 +564,16 @@ mod tests { #[test] fn empty_overlay_renders_authoritative_no_route_config() { let config = render(&overlay(Vec::new()), [("site-b", "203.0.113.7:9091")], &INPUTS); - assert!(config.candidates.is_empty()); - assert!(config.peers.is_empty()); - assert!(to_text(&config).expect("json").contains("\"candidates\": []")); + assert!(config.candidates.is_empty(), "the empty overlay withdraws every route"); + assert_eq!( + peer_sites(&config), + [("site-b", "203.0.113.7:9091")], + "signals pollers remain ready for restoration" + ); + assert!( + to_text(&config).expect("json").contains("\"candidates\": []"), + "the serialized revision carries an empty candidate list" + ); } #[test] @@ -682,10 +712,6 @@ mod tests { } #[test] - #[expect( - clippy::too_many_lines, - reason = "table-driven test covers both irrelevant and unsafe peer addresses" - )] fn unsafe_or_irrelevant_peers_are_refused() { let cases = [ ("loopback", "127.0.0.1:9091"), @@ -762,6 +788,19 @@ mod tests { ); } + #[test] + fn excluded_history_cannot_restore_an_authoritative_empty_revision() { + let mut withdrawn = overlay(Vec::new()); + withdrawn + .excluded + .push(cand("llama", "site-b", "provider-b", Some("none"))); + let config = render(&withdrawn, [], &INPUTS); + assert!( + config.candidates.is_empty(), + "an empty active overlay withdraws prior excluded history" + ); + } + #[test] fn authenticated_gateway_candidates_carry_overlay_id_and_mtls_allowlist() { let mut candidate = cand("llama", "site-b", "provider-b", None); diff --git a/tests/e2e/topologies/grid-provider-traffic/README.md b/tests/e2e/topologies/grid-provider-traffic/README.md index 61674c2cc..d21433f8e 100644 --- a/tests/e2e/topologies/grid-provider-traffic/README.md +++ b/tests/e2e/topologies/grid-provider-traffic/README.md @@ -175,7 +175,7 @@ scenarios. The qualification builds both consumers from compatible source under test. Deployments must upgrade and roll out every consumer before using this Grid -version, because older images (including the chart-default AI 0.4.0) reject +version, because images without empty-overlay support reject the authoritative empty snapshot. Full mode also exercises the generated consumer credential lifecycle in two @@ -210,9 +210,11 @@ A qualifying run demonstrates: - in full mode, a partial withdrawal serves only the remaining providers; - in full mode, last-route withdrawal publishes a new empty revision that the overlay-backed and embedded Grid gateway consumers both accept and serve; -- new and previously bound overlay-backed requests receive an unattributed - no-route response, with provider workloads still Ready and backend counters - unchanged; +- in full mode, new and previously bound overlay-backed requests and, when + embedded gateway setup succeeds, equivalent embedded Grid gateway requests + receive unattributed HTTP 404 responses (RFC 9110 §15.5.5); +- provider workloads are Ready before the no-route probes; backend request and + provider-gateway POST counters remain unchanged across the probes; - restoration publishes and serves a new revision and attributed requests resume; - teardown removes the clusters and shared network. diff --git a/xtask/src/env/certs.rs b/xtask/src/env/certs.rs index 153b04749..2b5cd1e77 100644 --- a/xtask/src/env/certs.rs +++ b/xtask/src/env/certs.rs @@ -68,7 +68,7 @@ pub(crate) fn generate_all_in_dir(cluster_names: &[String], dir: &Path) -> Resul let ca = load_or_generate_ca(dir)?; for name in cluster_names { - if ca_was_complete && identity_exists(&dir, name) && names_its_site(&dir, name)? { + if ca_was_complete && identity_exists(dir, name) && names_its_site(dir, name)? { restrict_private_key(&dir.join(format!("{name}-key.pem")))?; eprintln!(" reusing cert for {name}"); continue; @@ -448,7 +448,11 @@ mod tests { let fingerprint = site_certificate_fingerprint_in_dir("provider-b", test_dir.path()) .unwrap_or_else(|_| std::process::abort()); - assert_eq!(fingerprint.len(), 64); + assert_eq!( + fingerprint, + certificate_sha256(&test_dir.path().join("provider-b-cert.pem")).unwrap_or_else(|_| std::process::abort()), + "the helper must read the certificate from the supplied directory" + ); } #[test] diff --git a/xtask/src/env/image_overrides.rs b/xtask/src/env/image_overrides.rs index d9354addf..584213dea 100644 --- a/xtask/src/env/image_overrides.rs +++ b/xtask/src/env/image_overrides.rs @@ -164,13 +164,21 @@ pub(crate) fn should_skip_kind_image_loading() -> bool { /// Import a host image into a run-owned Kind node using only its linux/amd64 /// image content. This avoids OCI-index imports failing when the local Docker /// store has only the linux/amd64 child content. +#[expect( + clippy::too_many_lines, + reason = "ensures both image processes are reaped on every failure path" +)] pub(crate) fn load_docker_image_into_kind(image: &str, kind_name: &str) -> Result<(), Box> { let control_plane = format!("{kind_name}-control-plane"); let mut save = Command::new("docker") .args(["save", "--platform", "linux/amd64", image]) .stdout(Stdio::piped()) .spawn()?; - let save_stdout = save.stdout.take().ok_or("docker save did not provide stdout")?; + let Some(save_stdout) = save.stdout.take() else { + drop(save.kill()); + drop(save.wait()); + return Err("docker save did not provide stdout".into()); + }; let import_status = Command::new("docker") .args([ "exec", @@ -186,7 +194,15 @@ pub(crate) fn load_docker_image_into_kind(image: &str, kind_name: &str) -> Resul "-", ]) .stdin(save_stdout) - .status()?; + .status(); + let import_status = match import_status { + Ok(status) => status, + Err(error) => { + drop(save.kill()); + drop(save.wait()); + return Err(error.into()); + }, + }; let save_status = save.wait()?; if !save_status.success() { return Err(format!("docker save failed for {image}").into()); From 805b87e792b0ca7374734f2ffc28c9ac4ffc8514 Mon Sep 17 00:00:00 2001 From: Brent Salisbury Date: Thu, 8 Oct 2026 17:32:00 +0000 Subject: [PATCH 10/10] fix(grid): address routing review feedback Signed-off-by: Brent Salisbury --- gateway/ai-grid-filters/src/control.rs | 10 ++++------ gateway/ai-grid-filters/src/route.rs | 11 ++++------- operator/src/controller/grid_network.rs | 2 +- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index a081cc1f0..601bb03b7 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -95,7 +95,7 @@ pub(crate) struct Topology { load_window_ms: i64, /// Explicitly authenticated provider-gateway hop clusters. - provider_hop_clusters: BTreeSet, + provider_hop_clusters: Arc>, } impl Topology { @@ -117,7 +117,7 @@ impl Topology { base: Arc::from(base), local_site: Arc::from(config.local_site.as_str()), load_window_ms: config.load_window_ms, - provider_hop_clusters, + provider_hop_clusters: Arc::new(provider_hop_clusters), }) } @@ -148,7 +148,7 @@ impl Topology { Arc::clone(&self.local_site), &mut inputs, ); - ordered.provider_hop_clusters = Arc::new(self.provider_hop_clusters.clone()); + ordered.provider_hop_clusters = Arc::clone(&self.provider_hop_clusters); // Praxis demotes a cluster only once it has reported health. Without a registry the // gateway knows nothing about backends, so it demotes nothing rather than guessing. let ordered = if health.observed() { @@ -262,7 +262,7 @@ impl Control { Arc::clone(&topology.local_site), ) .published(&mut gauged.published); - cold_start.provider_hop_clusters = Arc::new(topology.provider_hop_clusters.clone()); + cold_start.provider_hop_clusters = Arc::clone(&topology.provider_hop_clusters); Ok(Self { store: Arc::new(LoadStore::with_combine( Duration::from_secs(config.window_secs), @@ -493,8 +493,6 @@ pub(crate) fn load_tag_key(affinity: &AffinitySettings) -> Result /// The topology of `config`, refusing what no retry can fix: a bad candidate, peer, or duplicate site. fn validate_config(config: &GridServingConfig) -> Result { - validate_local_site(&config.local_site)?; - validate_serving_candidates(config.candidates.clone())?; let topology = Topology::from_config(config)?; let mut sites = std::collections::HashSet::with_capacity(config.peers.len()); for peer in &config.peers { diff --git a/gateway/ai-grid-filters/src/route.rs b/gateway/ai-grid-filters/src/route.rs index e41838778..c9c477fc8 100644 --- a/gateway/ai-grid-filters/src/route.rs +++ b/gateway/ai-grid-filters/src/route.rs @@ -61,7 +61,7 @@ enum RouteDecision { } /// Resolve the empty revision before preserving a prior filter's selection. -fn route_decision(snapshot: &RouteSnapshot, _model: &str, earlier_selected: bool) -> RouteDecision { +fn route_decision(snapshot: &RouteSnapshot, earlier_selected: bool) -> RouteDecision { if snapshot.candidates.is_empty() { RouteDecision::NoRoute } else if earlier_selected { @@ -516,7 +516,7 @@ impl HttpFilter for GridSiteRouteFilter { .push(http::header::HeaderName::from_static(name)); } let snapshot = self.snapshot.load(); - match route_decision(&snapshot, "", ctx.cluster.is_some() || ctx.upstream.is_some()) { + match route_decision(&snapshot, ctx.cluster.is_some() || ctx.upstream.is_some()) { RouteDecision::NoRoute => { return Ok(refuse(Refused::NoRoute, "", self.turn.fetch_add(1, Ordering::Relaxed))); }, @@ -1927,15 +1927,12 @@ mod tests { #[test] fn empty_snapshot_rejects_even_with_a_preselected_cluster() { let empty = RouteSnapshot::from_static(Vec::new(), Arc::from("local")); - assert!(matches!(route_decision(&empty, "llama", true), RouteDecision::NoRoute)); + assert!(matches!(route_decision(&empty, true), RouteDecision::NoRoute)); let active = RouteSnapshot::from_static( one("llama", "east", "pool-a", AdmissionState::NewAndExisting), Arc::from("local"), ); - assert!(matches!( - route_decision(&active, "llama", true), - RouteDecision::KeepEarlier - )); + assert!(matches!(route_decision(&active, true), RouteDecision::KeepEarlier)); } } diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 562918f93..1a7b44018 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -2406,7 +2406,7 @@ async fn reconcile_routing_overlay_inner( .await { Ok(outcome) => { - if outcome.config_applied { + if outcome.config_applied && (!credential_bearing || cc.supports_projected_credentials) { consumer_statuses.push(consumer_config_status_rendered(gw_ref, cc, observed_generation)); } if let Some(status) = outcome.mount_status {