From 734d154b2399944538e359dbbf25964afcbdd1f1 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 17:54:19 -0400 Subject: [PATCH 01/10] fix(operator): mint the metrics scraper token unbound The scraper token was bound to the operator pod, which runs as the grid-operator ServiceAccount, and the API server binds a token only to a pod running as the token's own ServiceAccount. Every TokenRequest failed with 400, so no serviceAccountToken scrape ever ran, and the failure logged only at debug. Mint it unbound, keeping the 10-minute lifetime, the API server's default audience the EPP's TokenReview checks, and reuse at two thirds. A failed mint now logs at WARN when the error changes, and at INFO when minting recovers. The chart drops the pod name and uid env it read. The same mint code is in the base of #286, which rebases on this. Signed-off-by: Sam Batschelet --- .../grid-operator/templates/deployment.yaml | 9 -- docs/architecture/crds.md | 6 +- operator/src/metrics_token.rs | 92 ++++++++++++++----- 3 files changed, 71 insertions(+), 36 deletions(-) diff --git a/charts/grid-operator/templates/deployment.yaml b/charts/grid-operator/templates/deployment.yaml index 5968ff6c..6524fcd8 100644 --- a/charts/grid-operator/templates/deployment.yaml +++ b/charts/grid-operator/templates/deployment.yaml @@ -108,15 +108,6 @@ spec: value: {{ printf "%s-metrics-scraper" (include "grid-operator.fullname" .) | quote }} - name: GRID_METRICS_SCRAPER_NAMESPACE value: {{ .Release.Namespace | quote }} - # Binds each scrape token to this pod, so it stops working when the pod goes. - - name: GRID_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: GRID_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid {{- end }} {{- if (.Values.signals).enabled }} - name: GRID_SIGNALS_LOCAL_ADDR diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 807b1bed..724aebfa 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -739,8 +739,10 @@ Whoever runs the EPP receives the token, and it is valid against the API server, the operator never sends its own token. The grid-operator chart creates a scraper ServiceAccount allowed only `get` on the nonResourceURL `/metrics` (`rbac.metricsScraper`, default `true`), and the operator mints a 10-minute token for it -with the TokenRequest API, bound to the operator Pod, reusing it until two thirds of its -lifetime has passed (about 400 seconds). The token is never logged. +with the TokenRequest API, reusing it until two thirds of its lifetime has passed (about +400 seconds). The token is not bound to a Pod: the API server binds a token only to a Pod +running as the token's own ServiceAccount, and the operator runs as another. The token is +never logged. A failed mint logs at WARN when the error changes. A credential goes only to a host proven by the CA `metricsConfig.tls` names: with `auth` set, an `https://` endpoint without `tls` is refused rather than trusted through diff --git a/operator/src/metrics_token.rs b/operator/src/metrics_token.rs index b4ffad24..1a9be1ff 100644 --- a/operator/src/metrics_token.rs +++ b/operator/src/metrics_token.rs @@ -11,7 +11,7 @@ use std::{ }; use k8s_openapi::api::{ - authentication::v1::{BoundObjectReference, TokenRequest, TokenRequestSpec}, + authentication::v1::{TokenRequest, TokenRequestSpec}, core::v1::ServiceAccount, }; use kube::api::{Api, PostParams}; @@ -32,10 +32,6 @@ const RETRY_BACKOFF: Duration = Duration::from_secs(30); const SERVICE_ACCOUNT_ENV: &str = "GRID_METRICS_SCRAPER_SERVICE_ACCOUNT"; /// The namespace of [`SERVICE_ACCOUNT_ENV`]. const NAMESPACE_ENV: &str = "GRID_METRICS_SCRAPER_NAMESPACE"; -/// This pod's name and uid, from the downward API, to bind each token to the pod. -const POD_NAME_ENV: &str = "GRID_POD_NAME"; -/// See [`POD_NAME_ENV`]. -const POD_UID_ENV: &str = "GRID_POD_UID"; /// A minted token and its timeline. struct Cached { @@ -56,6 +52,8 @@ struct State { cached: Option, /// After a failed mint with no usable token, the earliest moment to try again. no_mint_before: Option, + /// The last mint failure logged, so a failure that repeats logs once. + failing: Option, } impl State { @@ -67,6 +65,9 @@ impl State { ) -> Result, MetricsScrapeError> { let error = match minted { Ok((token, lifetime)) => { + if self.failing.take().is_some() { + tracing::info!("metrics scraper token minted again"); + } *self = Self { cached: Some(Cached { token: token.clone(), @@ -75,19 +76,31 @@ impl State { retry_at: None, }), no_mint_before: None, + failing: None, }; return Ok(token); }, Err(error) => error, }; + self.note_failure(&error, now); if let Some(cached) = self.cached.as_mut().filter(|cached| now < cached.expires_at) { - tracing::warn!(%error, "metrics scraper token refresh failed; using the held token until it expires"); cached.retry_at = Some(now + RETRY_BACKOFF); return Ok(cached.token.clone()); } self.no_mint_before = Some(now + RETRY_BACKOFF); Err(error) } + + /// Log a mint failure at WARN once, until a different one or a success. + fn note_failure(&mut self, error: &MetricsScrapeError, now: Instant) { + let message = error.to_string(); + if self.failing.as_deref() == Some(message.as_str()) { + return; + } + let held = self.cached.as_ref().is_some_and(|cached| now < cached.expires_at); + tracing::warn!(%error, held, "metrics scraper token mint failed"); + self.failing = Some(message); + } } /// What a scrape does with the cached token at `now`. @@ -115,16 +128,20 @@ fn token_plan(state: &State, now: Instant) -> Plan { /// The current token, shared by every scrape. static CACHE: LazyLock> = LazyLock::new(|| tokio::sync::Mutex::new(State::default())); -/// This pod, when the downward API names it, as the object each token is bound to. -fn bound_pod() -> Option { - let name = std::env::var(POD_NAME_ENV).ok().filter(|v| !v.trim().is_empty())?; - let uid = std::env::var(POD_UID_ENV).ok().filter(|v| !v.trim().is_empty())?; - Some(BoundObjectReference { - api_version: Some("v1".to_owned()), - kind: Some("Pod".to_owned()), - name: Some(name), - uid: Some(uid), - }) +/// The `TokenRequest` for a scraper token. +/// +/// Unbound: the API server binds a token only to a pod running as the token's own +/// `ServiceAccount`, and the operator runs as another. +fn token_request() -> TokenRequest { + TokenRequest { + spec: TokenRequestSpec { + // The API server's own audience: an EPP's `TokenReview` checks no other. + audiences: Vec::new(), + expiration_seconds: i64::try_from(TOKEN_LIFETIME.as_secs()).ok(), + bound_object_ref: None, + }, + ..TokenRequest::default() + } } /// The configured scraper `ServiceAccount`, read once. @@ -147,15 +164,7 @@ async fn mint( name: &str, namespace: &str, ) -> Result<(Zeroizing, Duration), MetricsScrapeError> { - let request = TokenRequest { - spec: TokenRequestSpec { - // The API server's own audience: an EPP's `TokenReview` checks no other. - audiences: Vec::new(), - expiration_seconds: i64::try_from(TOKEN_LIFETIME.as_secs()).ok(), - bound_object_ref: bound_pod(), - }, - ..TokenRequest::default() - }; + let request = token_request(); let api = Api::::namespaced(client.clone(), namespace); let issued = tokio::time::timeout( MINT_TIMEOUT, @@ -224,6 +233,7 @@ mod tests { retry_at: retry_in.map(|secs| now + Duration::from_secs(secs)), }), no_mint_before: None, + failing: None, } } @@ -256,6 +266,7 @@ mod tests { let failed = State { cached: None, no_mint_before: Some(now + Duration::from_secs(30)), + failing: None, }; assert_eq!(token_plan(&failed, now), Plan::Fail, "backing off"); assert_eq!( @@ -265,6 +276,37 @@ mod tests { ); } + #[test] + fn a_token_is_unbound_for_the_api_servers_audience() { + let spec = token_request().spec; + assert!( + spec.bound_object_ref.is_none(), + "a pod running as another ServiceAccount cannot hold the binding" + ); + assert!(spec.audiences.is_empty(), "the API server's own audience"); + assert_eq!(spec.expiration_seconds, Some(600)); + } + + #[test] + fn a_repeated_mint_failure_logs_once_and_a_success_clears_it() { + let now = Instant::now(); + let mut state = State::default(); + let fail = || Err(MetricsScrapeError::Credential("refused".to_owned())); + assert_eq!(state.record(fail(), now).map(drop).ok(), None); + assert!( + state + .failing + .as_deref() + .is_some_and(|logged| logged.contains("refused")) + ); + let logged = state.failing.clone(); + assert_eq!(state.record(fail(), now).map(drop).ok(), None); + assert_eq!(state.failing, logged, "the same failure is not logged again"); + let token = state.record(Ok((Zeroizing::new("t".to_owned()), TOKEN_LIFETIME)), now); + assert_eq!(token.as_deref().map(String::as_str).ok(), Some("t")); + assert!(state.failing.is_none()); + } + #[test] fn the_api_servers_expiry_wins_over_the_requested_lifetime() { let now = UNIX_EPOCH + Duration::from_secs(1_000_000); From cd56658afcd8ae04cac6b4e4701d684ca483b8b1 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sat, 3 Oct 2026 14:03:54 -0400 Subject: [PATCH 02/10] feat(operator)!: resolve provider readiness The operator decides from each scrape of a provider's EPP whether it can serve now, publishes the verdict as grid_provider_ready, writes it to the provider's Ready condition on change, and shows it as one STATUS column, like nodes. Signed-off-by: Sam Batschelet --- .../templates/crds/inferenceprovider.yaml | 70 +++ deploy/crds/inferenceprovider.yaml | 70 +++ docs/architecture/crds.md | 29 + docs/architecture/polling-metrics.md | 23 + operator/src/controller/grid_network.rs | 285 +++++++++- operator/src/controller/inference_provider.rs | 21 + operator/src/crd/inference_provider.rs | 48 +- operator/src/lib.rs | 2 + operator/src/main.rs | 1 + operator/src/readiness.rs | 535 ++++++++++++++++++ operator/src/resources/consumer_config.rs | 2 + operator/src/resources/overlay_bridge.rs | 1 + operator/src/resources/overlay_envelope.rs | 2 + operator/src/resources/provider_metrics.rs | 114 ++-- operator/src/resources/routing_overlay.rs | 10 +- operator/src/resources/serving_config.rs | 140 ++++- 16 files changed, 1289 insertions(+), 64 deletions(-) create mode 100644 operator/src/readiness.rs diff --git a/charts/grid-operator/templates/crds/inferenceprovider.yaml b/charts/grid-operator/templates/crds/inferenceprovider.yaml index 57e33f9a..613adc04 100644 --- a/charts/grid-operator/templates/crds/inferenceprovider.yaml +++ b/charts/grid-operator/templates/crds/inferenceprovider.yaml @@ -24,8 +24,19 @@ spec: - jsonPath: .spec.providerKind name: Provider type: string + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + priority: 1 + type: string - jsonPath: .status.phase name: Phase + priority: 1 type: string name: v1alpha1 schema: @@ -407,6 +418,14 @@ spec: description: Metric name for normalised queue depth (0.0–1.0). nullable: true type: string + readyEndpoints: + description: |- + Metric name counting the pool's ready endpoints, read for the `Ready` condition. + + Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`. + Filtered by `poolName` when set. Zero marks the provider not ready. + nullable: true + type: string type: object staleMetricsSeconds: description: |- @@ -764,6 +783,46 @@ spec: description: Observed status of an [`InferenceProvider`]. nullable: true properties: + conditions: + description: |- + Observed conditions. `Ready` says whether the provider can currently serve a request. + + Written by the operator's signals loop, never by provider reconciliation. + items: + description: One observed condition, shaped like `metav1.Condition`. + properties: + lastTransitionTime: + description: When `status` last changed, RFC 3339. + format: date-time + type: string + message: + default: "" + description: Human-readable detail. + type: string + observedGeneration: + description: The `metadata.generation` this was computed against. + format: int64 + nullable: true + type: integer + reason: + description: CamelCase reason for the status. + type: string + status: + description: '`True`, `False`, or `Unknown`.' + type: string + type: + description: Condition type, such as `Ready`. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map matchingSites: default: [] description: Sites matched by the site selector. @@ -811,6 +870,17 @@ spec: `HealthCheckTlsIdentityMismatch` nullable: true type: string + state: + description: |- + `Ready`, `NotReady`, or `Unknown`: the `Ready` condition's status, for display. + + Written with the condition, never by provider reconciliation. + enum: + - Ready + - NotReady + - Unknown + nullable: true + type: string type: object required: - spec diff --git a/deploy/crds/inferenceprovider.yaml b/deploy/crds/inferenceprovider.yaml index 910be4c9..1db5b55a 100644 --- a/deploy/crds/inferenceprovider.yaml +++ b/deploy/crds/inferenceprovider.yaml @@ -16,8 +16,19 @@ spec: - jsonPath: .spec.providerKind name: Provider type: string + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + priority: 1 + type: string - jsonPath: .status.phase name: Phase + priority: 1 type: string name: v1alpha1 schema: @@ -399,6 +410,14 @@ spec: description: Metric name for normalised queue depth (0.0–1.0). nullable: true type: string + readyEndpoints: + description: |- + Metric name counting the pool's ready endpoints, read for the `Ready` condition. + + Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`. + Filtered by `poolName` when set. Zero marks the provider not ready. + nullable: true + type: string type: object staleMetricsSeconds: description: |- @@ -756,6 +775,46 @@ spec: description: Observed status of an [`InferenceProvider`]. nullable: true properties: + conditions: + description: |- + Observed conditions. `Ready` says whether the provider can currently serve a request. + + Written by the operator's signals loop, never by provider reconciliation. + items: + description: One observed condition, shaped like `metav1.Condition`. + properties: + lastTransitionTime: + description: When `status` last changed, RFC 3339. + format: date-time + type: string + message: + default: "" + description: Human-readable detail. + type: string + observedGeneration: + description: The `metadata.generation` this was computed against. + format: int64 + nullable: true + type: integer + reason: + description: CamelCase reason for the status. + type: string + status: + description: '`True`, `False`, or `Unknown`.' + type: string + type: + description: Condition type, such as `Ready`. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map matchingSites: default: [] description: Sites matched by the site selector. @@ -803,6 +862,17 @@ spec: `HealthCheckTlsIdentityMismatch` nullable: true type: string + state: + description: |- + `Ready`, `NotReady`, or `Unknown`: the `Ready` condition's status, for display. + + Written with the condition, never by provider reconciliation. + enum: + - Ready + - NotReady + - Unknown + nullable: true + type: string type: object required: - spec diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 724aebfa..b18cb62f 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -563,6 +563,34 @@ credential projection can become available. **Phases**: Pending → Available → Degraded → Unavailable +**Readiness**: `status.conditions` carries a `Ready` condition, written by the +operator's signals loop from each scrape of the provider's metrics. Readiness is +a condition, not a phase, so `phase` is unchanged by it. + +| Status | Reason | When | +|---|---|---| +| `True` | `Ready` | The latest scrape succeeded with at least one ready endpoint, or exposed no count. | +| `False` | `NoReadyEndpoints` | Two consecutive scrapes counted zero ready endpoints. | +| `False` | `MetricsUnreachable` | No scrape succeeded within `staleMetricsSeconds`, and the latest failed. | +| `False` | `MetricsStale` | No scrape succeeded within `staleMetricsSeconds`. | +| `False` | `ProviderUnavailable` | The provider is `Unavailable`. | +| `Unknown` | `MetricsNotConfigured` | No `metricsConfig`, so readiness cannot be read. | + +A provider whose `Ready` is `False` is excluded from routing: its site publishes +`grid_provider_ready 0`, and its serving config entry carries `admission: none`. +See [Polling Cross-Site Load Signals](polling-metrics.md#provider-readiness). + +`status.state` repeats the condition's status as one word for display, `Ready`, +`NotReady`, or `Unknown`, so `kubectl get inferenceproviders` reads like +`kubectl get nodes`. The STATUS column shows it. `-o wide` adds REASON, the +condition's reason, and PHASE. + +```text +NAME PROVIDER STATUS AGE +qwen3-site-a self_hosted Ready 3d +qwen3-site-b self_hosted NotReady 3d +``` + `spec.capacityWeight` is an optional positive relative provider capacity from `1` through `1000`, used only with `GridNetwork.spec.selectionPolicy.mode: weightedRandom` and `placementPolicy.strategy: static`. If omitted, the @@ -651,6 +679,7 @@ routing architecture for full semantics. | `prefixCacheHitRatio` | Prefix-cache hit ratio from `0.0` to `1.0`. | | `errorRate` | Error rate from `0.0` to `1.0`. | | `healthy` | Health gauge interpreted by the metrics parser. | +| `readyEndpoints` | Ready endpoints in the pool, read for the `Ready` condition. Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`, filtered by `poolName`. | #### TLS and mTLS diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index c026acd9..0c32a566 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -50,6 +50,28 @@ the store and produces a candidate list ordered least-loaded-first. The request path reads one ordered snapshot and takes the front admitted candidate. The request path does not read raw signals or compute load. It reads resolved order. +## Provider Readiness + +Each site's operator decides whether each of its providers can serve now and +publishes the verdict as `grid_provider_ready{grid_site,grid_provider}`: 1 when +ready, 0 when not. A provider is not ready when its EPP reports zero ready +endpoints for two scrapes in a row, when no scrape has succeeded within +`staleMetricsSeconds` (half the signal TTL when unset), or when the provider is +`Unavailable`. The same verdict is the provider's `Ready` condition. + +The gateway reads the latest readiness sample for each candidate when it orders +the snapshot. A candidate whose latest sample is 0 is excluded, so a site is +dropped within one poll of its operator deciding, and readmitted within one poll +of it recovering. A missing series reads as ready, so a site whose operator +predates readiness is still routed. The serving config carries the same verdict +for this site's own providers as `admission: none`. + +When every candidate for a model is excluded, the gateway answers 503 with +`Retry-After`, not 404: the model exists but cannot be served now. + +With the defaults of a 5 s scrape and a 5 s poll, exclusion takes at most about +15 s and rejoin about 10 s. + ## Failure Behavior | Condition | Signal produced | Routing effect | @@ -58,6 +80,7 @@ request path does not read raw signals or compute load. It reads resolved order. | Readings all stale (older than the window) | Same as no reading. | The candidate sorts last until a fresh reading arrives. | | Peer unreachable or slow | The poll returns an error and no reading is written. | The candidate ages out of the window and then sorts last. The poll loop continues, and one unreachable peer does not wedge the others. | | Peer presents an untrusted or mismatched certificate | The connection is refused, so no reading is written. | The peer contributes nothing to the order. | +| Peer reports `grid_provider_ready 0` | The candidate is excluded. | It takes no new requests until a later reading says 1. If every candidate is excluded, the model answers 503. | Loss of signal degrades to "least preferred," never to "silently treated as idle." A drained burst stays penalized until it ages out of the window rather diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 9438805c..53f550a5 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -34,6 +34,7 @@ use crate::{ inference_provider::InferenceProvider, }, error::OperatorError, + readiness, resources::{ consumer_config::{self, ConsumerConfigError}, overlay_envelope, provider_admission, provider_metrics, routing_overlay, secret, @@ -112,6 +113,12 @@ pub struct OperatorCtx { /// Served-model sets discovered from this site's providers. pub(crate) served_models: served_models::ServedModelStore, + /// Each provider's latest scrape, from which its readiness is resolved. + pub(crate) readiness: readiness::ReadinessStore, + + /// How often the signals loop scrapes providers; a readiness window is at least two. + scrape_interval: Duration, + /// Signal transport resolved once at startup: gossip or poll. /// /// Under poll the operator stops carrying metrics in gossip and scoring the @@ -275,6 +282,8 @@ impl OperatorCtx { peers: signals::SignalStore::new(), signals: signals::SignalStore::new(), served_models: served_models::ServedModelStore::new(), + readiness: readiness::ReadinessStore::default(), + scrape_interval: Duration::ZERO, signal_mode, serving_writes: WriteGate::default(), peer_settings: PeerSettings::default(), @@ -357,6 +366,13 @@ impl OperatorCtx { self.site_name.as_deref() } + /// Record how often the signals loop scrapes providers. + #[must_use] + pub const fn with_scrape_interval(mut self, interval: Duration) -> Self { + self.scrape_interval = interval; + self + } + /// Name the certificate the operator issues itself after this site, not the network. #[must_use] pub fn with_site_name(mut self, site_name: Option) -> Self { @@ -419,11 +435,207 @@ pub async fn refresh_signals(ctx: &OperatorCtx, client: &Client, network_name: & let providers = list_all_inference_providers(client).await?; ctx.signals.set_access(signal_access(&providers)); Box::pin(register_peers(ctx, client)).await?; - let collected = provider_metrics::collect_provider_signals(network_name, &providers, Some(client)).await; + Box::pin(provider_metrics::collect_provider_signals( + network_name, + &providers, + Some(client), + &ctx.readiness, + )) + .await; + let collected = Box::pin(resolve_readiness(ctx, client, &providers, network_name)).await; publish_signals(ctx, collected); Ok(()) } +/// Resolve each provider's readiness, write its `Ready` condition, and return what to publish. +async fn resolve_readiness( + ctx: &OperatorCtx, + client: &Client, + providers: &[InferenceProvider], + network_name: &str, +) -> HashMap> { + ctx.readiness.retain( + &providers + .iter() + .filter_map(|p| { + provider_metrics::signal_scrape_plan(p) + .map(|plan| readiness::key(&p.spec.grid_network_ref, plan.identity)) + }) + .collect(), + ); + let now = Instant::now(); + let mut collected = HashMap::new(); + for provider in providers.iter().filter(|p| p.spec.grid_network_ref == network_name) { + let Some(entry) = readiness_entry(ctx, provider, now) else { + continue; + }; + if let Err(error) = Box::pin(apply_ready_condition(client, provider, &entry.verdict)).await { + tracing::warn!(provider = entry.identity, %error, "provider Ready condition not written"); + } + if let Some(published) = entry.published { + collected.insert(entry.identity.to_owned(), published); + } + } + collected +} + +/// One provider's readiness and the signals to publish for it. +struct ReadinessEntry<'provider> { + /// Routing identity. + identity: &'provider str, + /// The verdict. + verdict: readiness::Verdict, + /// Signals to publish, `None` for a provider without metrics: absent reads as ready. + published: Option>, +} + +/// A provider's readiness entry, `None` for no identity or not yet judged. +/// +/// A provider not scraped for readiness, with no metrics config or no usable +/// endpoint, is `Unknown` and publishes nothing: absent reads as ready. +fn readiness_entry<'provider>( + ctx: &OperatorCtx, + provider: &'provider InferenceProvider, + now: Instant, +) -> Option> { + let identity = routing_overlay::routing_identity(provider)?; + if provider_metrics::signal_scrape_plan(provider).is_none() { + return Some(ReadinessEntry { + identity, + verdict: readiness::Verdict { + reason: readiness::Reason::MetricsNotConfigured, + message: "no metricsConfig endpoint to read readiness from".to_owned(), + }, + published: None, + }); + } + let verdict = provider_readiness(ctx, provider, now)?; + let ready = ready_sample(&verdict); + // A fresh scrape republishes with the verdict. Without one, the last published + // entry ages on its own, unless the verdict turned not ready. + let fresh = ctx + .readiness + .take_fresh(&readiness::key(&provider.spec.grid_network_ref, identity)); + let published = match fresh { + Some(mut held) => { + held.push(ready); + Some(held) + }, + None if verdict.reason.excludes() => Some(vec![ready]), + None => None, + }; + Some(ReadinessEntry { + identity, + verdict, + published, + }) +} + +/// The `grid_provider_ready` sample for `verdict`: 1 when it serves, 0 when not. +fn ready_sample(verdict: &readiness::Verdict) -> signals::Observation { + signals::Observation { + metric: readiness::READY_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value: if verdict.reason.excludes() { 0.0 } else { 1.0 }, + timestamp_ms: None, + } +} + +/// How long a provider's last good scrape stands. +/// +/// `staleMetricsSeconds`, else half the signal TTL, and never under two scrape +/// intervals, so a provider is not judged stale between successful scrapes. +fn readiness_stale_after(ctx: &OperatorCtx, provider: &InferenceProvider) -> Duration { + provider + .spec + .metrics_config + .as_ref() + .and_then(|mc| mc.stale_metrics_seconds) + .filter(|secs| *secs > 0) + .map_or_else(|| site_signals_ttl() / 2, |secs| Duration::from_secs(secs.into())) + .max(ctx.scrape_interval.saturating_mul(2)) +} + +/// The provider's readiness verdict, `None` when it is not scraped for readiness or not yet judged. +pub(crate) fn provider_readiness( + ctx: &OperatorCtx, + provider: &InferenceProvider, + now: Instant, +) -> Option { + let plan = provider_metrics::signal_scrape_plan(provider)?; + let unavailable = provider + .status + .as_ref() + .is_some_and(|status| status.phase == crate::crd::inference_provider::ProviderPhase::Unavailable); + ctx.readiness.verdict( + &readiness::key(&provider.spec.grid_network_ref, plan.identity), + unavailable, + readiness_stale_after(ctx, provider), + now, + ) +} + +/// Write the provider's `Ready` condition when its status or reason changed. +async fn apply_ready_condition( + client: &Client, + provider: &InferenceProvider, + verdict: &readiness::Verdict, +) -> Result<(), OperatorError> { + let Some((name, patch)) = ready_condition_patch(provider, verdict) else { + return Ok(()); + }; + let api: Api = Api::all(client.clone()); + Box::pin(api.patch_status( + name, + &PatchParams::apply(READINESS_FIELD_MANAGER).force(), + &Patch::Apply(patch), + )) + .await?; + tracing::info!( + provider = name, + status = verdict.reason.status(), + reason = verdict.reason.as_str(), + message = %verdict.message, + "provider readiness changed" + ); + Ok(()) +} + +/// The status patch carrying the provider's new `Ready` condition, `None` when unchanged. +fn ready_condition_patch<'provider>( + provider: &'provider InferenceProvider, + verdict: &readiness::Verdict, +) -> Option<(&'provider str, serde_json::Value)> { + let name = provider.metadata.name.as_deref()?; + let status = provider.status.as_ref(); + let current = status.map_or(&[][..], |status| status.conditions.as_slice()); + let shown = verdict.reason.display(); + let now = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .ok()?; + // Apply always carries both fields this manager owns: one it left out would be removed. + let condition = match readiness::ready_condition(current, verdict, &now, provider.metadata.generation) { + Some(condition) => condition, + None if status.and_then(|status| status.state.as_deref()) == Some(shown) => return None, + None => current + .iter() + .find(|condition| condition.type_ == readiness::READY_CONDITION)? + .clone(), + }; + Some(( + name, + serde_json::json!({ + "apiVersion": "grid.praxis.fast/v1alpha1", + "kind": "InferenceProvider", + "metadata": { "name": name }, + "status": { "conditions": [condition], "state": shown } + }), + )) +} + +/// Server-side apply manager for the `Ready` condition, apart from provider reconciliation. +const READINESS_FIELD_MANAGER: &str = "grid-operator-readiness"; + /// Poll this site's providers once and hold the models they serve. /// /// Called from its own loop rather than from reconcile, like @@ -939,6 +1151,13 @@ pub async fn reconcile(network: Arc, ctx: Arc) -> Resu memory.evaluate(&memory_key, observation, admission_policy, now), provider.spec.traffic_policy.as_ref().is_some_and(|p| p.drain), ); + // Readiness gates admission whatever the scoring strategy, NoMetrics included. + let not_ready = provider_readiness(&ctx, provider, now).is_some_and(|verdict| verdict.reason.excludes()); + let state = if not_ready { + crate::resources::geography::AdmissionState::Excluded + } else { + state + }; admission_keys.push(memory_key); admission_states.insert(identity, state); } @@ -3776,6 +3995,69 @@ fn parse_metrics_refresh_interval(value: &str) -> Result InferenceProvider { + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis.fast/v1alpha1", + "kind": "InferenceProvider", + "metadata": { "name": "qwen3-site-b", "generation": 1 }, + "spec": { + "gridNetworkRef": "grid", + "providerKind": "self_hosted", + "backendKind": "local", + "endpoint": "http://localhost:8000", + "models": [] + }, + "status": status + })) + .expect("provider") + } + + fn ready_condition_json(status: &str, reason: &str) -> serde_json::Value { + serde_json::json!({ + "type": "Ready", "status": status, "reason": reason, "message": "m", + "lastTransitionTime": "2026-10-03T00:00:00Z", "observedGeneration": 1 + }) + } + + #[test] + #[expect( + clippy::expect_used, + clippy::indexing_slicing, + clippy::shadow_unrelated, + reason = "test assertions on a JSON patch" + )] + fn the_status_column_is_written_with_the_condition_and_backfilled_once() { + let down = readiness::Verdict { + reason: readiness::Reason::NoReadyEndpoints, + message: "0 ready endpoints".to_owned(), + }; + // The condition already says so, but STATE was never written: backfill it. + let unshown = provider_with_status(&serde_json::json!({ + "conditions": [ready_condition_json("False", "NoReadyEndpoints")] + })); + let (_, patch) = ready_condition_patch(&unshown, &down).expect("state is backfilled"); + assert_eq!(patch["status"]["state"], "NotReady"); + assert_eq!( + patch["status"]["conditions"][0]["lastTransitionTime"], "2026-10-03T00:00:00Z", + "the held condition rides along unchanged, so apply does not drop it" + ); + // Both already current: nothing to write. + let shown = provider_with_status(&serde_json::json!({ + "conditions": [ready_condition_json("False", "NoReadyEndpoints")], + "state": "NotReady" + })); + assert!(ready_condition_patch(&shown, &down).is_none()); + // A transition writes both. + let up = readiness::Verdict { + reason: readiness::Reason::Ready, + message: "1 ready endpoints".to_owned(), + }; + let (_, patch) = ready_condition_patch(&shown, &up).expect("a transition is written"); + assert_eq!(patch["status"]["state"], "Ready"); + assert_eq!(patch["status"]["conditions"][0]["status"], "True"); + } + fn network_with_modes(spec: &serde_json::Value) -> GridNetwork { serde_json::from_value(serde_json::json!({ "apiVersion": "grid.praxis.fast/v1alpha1", @@ -6610,6 +6892,7 @@ mod tests { network: "net".to_owned(), local_site: "site".to_owned(), candidates: Vec::new(), + excluded: Vec::new(), selection_policy: None, generated_at: Some("2026-07-29T01:00:00Z".to_owned()), }, diff --git a/operator/src/controller/inference_provider.rs b/operator/src/controller/inference_provider.rs index 9c859c85..21a8ba52 100644 --- a/operator/src/controller/inference_provider.rs +++ b/operator/src/controller/inference_provider.rs @@ -699,6 +699,8 @@ async fn update_status( ModelDiscoveryConfig::OpenAiModels(openai) => openai.effective_url(&provider.spec.endpoint), }); let status = InferenceProviderStatus { + conditions: Vec::new(), + state: None, matching_sites, model_discovery_error: None, model_discovery_url, @@ -741,6 +743,8 @@ mod tests { #[test] fn reconciler_status_matches_only_reconciler_fields() { let baseline = InferenceProviderStatus { + conditions: Vec::new(), + state: None, matching_sites: vec!["site-a".to_owned()], model_discovery_error: None, model_discovery_url: None, @@ -769,6 +773,23 @@ mod tests { assert!(!baseline.matches_reconciler_status(&changed_discovery_url)); } + #[test] + fn a_conditions_change_is_not_the_reconcilers_to_write() { + let baseline = InferenceProviderStatus::default(); + let changed_conditions = InferenceProviderStatus { + conditions: vec![crate::crd::inference_provider::Condition { + type_: "Ready".to_owned(), + status: "False".to_owned(), + ..Default::default() + }], + ..baseline.clone() + }; + assert!( + baseline.matches_reconciler_status(&changed_conditions), + "the signals loop owns conditions" + ); + } + // ----------------------------------------------------------------------- // Test utilities // ----------------------------------------------------------------------- diff --git a/operator/src/crd/inference_provider.rs b/operator/src/crd/inference_provider.rs index e4ae8e90..7f9abf36 100644 --- a/operator/src/crd/inference_provider.rs +++ b/operator/src/crd/inference_provider.rs @@ -29,7 +29,10 @@ use super::{ status = "InferenceProviderStatus", namespaced = false, printcolumn = r#"{"name":"Provider","type":"string","jsonPath":".spec.providerKind"}"#, - printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"# + printcolumn = r#"{"name":"Status","type":"string","jsonPath":".status.state"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#, + printcolumn = r#"{"name":"Reason","type":"string","jsonPath":".status.conditions[?(@.type==\"Ready\")].reason","priority":1}"#, + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase","priority":1}"# )] #[serde(rename_all = "camelCase")] pub struct InferenceProviderSpec { @@ -443,6 +446,13 @@ pub struct MetricSignalNames { /// Metric name for a health gauge (any positive value = healthy). #[serde(skip_serializing_if = "Option::is_none")] pub healthy: Option, + + /// Metric name counting the pool's ready endpoints, read for the `Ready` condition. + /// + /// Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`. + /// Filtered by `poolName` when set. Zero marks the provider not ready. + #[serde(skip_serializing_if = "Option::is_none")] + pub ready_endpoints: Option, } /// Returns the default metrics scrape path. @@ -589,6 +599,20 @@ fn default_models_path() -> String { #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct InferenceProviderStatus { + /// Observed conditions. `Ready` says whether the provider can currently serve a request. + /// + /// Written by the operator's signals loop, never by provider reconciliation. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))] + pub conditions: Vec, + + /// `Ready`, `NotReady`, or `Unknown`: the `Ready` condition's status, for display. + /// + /// Written with the condition, never by provider reconciliation. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(extend("enum" = ["Ready", "NotReady", "Unknown"]))] + pub state: Option, + /// Sites matched by the site selector. #[serde(default)] pub matching_sites: Vec, @@ -638,6 +662,28 @@ impl InferenceProviderStatus { } } +/// One observed condition, shaped like `metav1.Condition`. +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Condition { + /// Condition type, such as `Ready`. + #[serde(rename = "type")] + pub type_: String, + /// `True`, `False`, or `Unknown`. + pub status: String, + /// CamelCase reason for the status. + pub reason: String, + /// Human-readable detail. + #[serde(default)] + pub message: String, + /// When `status` last changed, RFC 3339. + #[schemars(extend("format" = "date-time"))] + pub last_transition_time: String, + /// The `metadata.generation` this was computed against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, +} + /// Lifecycle phase of a provider resource. #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub enum ProviderPhase { diff --git a/operator/src/lib.rs b/operator/src/lib.rs index b38bf14a..4cb3fdc3 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -41,6 +41,8 @@ pub mod metrics_scraper; pub mod metrics_tls; /// Short-lived tokens for the metrics scraper ServiceAccount. pub(crate) mod metrics_token; +/// Provider readiness resolved from scraped metrics. +pub mod readiness; /// Kubernetes resource builders. pub mod resources; pub mod served_models; diff --git a/operator/src/main.rs b/operator/src/main.rs index 6ac0a211..d5a2a5e3 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -178,6 +178,7 @@ async fn main() { .with_declared_trust(declared_trust) .with_rotation(rotation_running) .with_site_name(config.swim.site_name.clone()) + .with_scrape_interval(config.signals.scrape_interval()) .hold_membership(), ); diff --git a/operator/src/readiness.rs b/operator/src/readiness.rs new file mode 100644 index 00000000..adcc533f --- /dev/null +++ b/operator/src/readiness.rs @@ -0,0 +1,535 @@ +//! Whether each provider can serve a request now, resolved from its scraped metrics. +//! +//! The signals loop records each scrape here. The verdict feeds three readers: the +//! provider's `Ready` condition, the `grid_provider_ready` series peers poll, and +//! admission in the serving config. + +use std::{ + collections::HashMap, + sync::{Mutex, PoisonError}, + time::{Duration, Instant}, +}; + +use crate::{crd::inference_provider::Condition, signals::Observation}; + +/// Metric names that count a pool's ready endpoints, preferred first. +pub(crate) const DEFAULT_READY_ENDPOINTS: [&str; 2] = ["llm_d_epp_ready_endpoints", "inference_pool_ready_pods"]; + +/// Consecutive scrapes reading zero ready endpoints before a provider is not ready, +/// and reading some before it is ready again. +/// +/// One reading can catch a pod between its metrics going stale and a fresh one, and a +/// pool that just drained can read ready once before its first request lands. +pub(crate) const STREAK: u32 = 2; + +/// The series this site publishes per provider: 1 when ready, 0 when not. +pub const READY_SIGNAL: &str = "grid_provider_ready"; + +/// The `Ready` condition's status and reason for one provider. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reason { + /// Scraped recently with at least one ready endpoint. + Ready, + /// The latest scrape counted zero ready endpoints. + NoReadyEndpoints, + /// No successful scrape within the staleness window. + MetricsStale, + /// No successful scrape within the window, and the latest attempt failed. + MetricsUnreachable, + /// The provider itself is `Unavailable`. + ProviderUnavailable, + /// The provider declares no metrics, so readiness is unknown. + MetricsNotConfigured, +} + +impl Reason { + /// The condition reason as written to status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Ready => "Ready", + Self::NoReadyEndpoints => "NoReadyEndpoints", + Self::MetricsStale => "MetricsStale", + Self::MetricsUnreachable => "MetricsUnreachable", + Self::ProviderUnavailable => "ProviderUnavailable", + Self::MetricsNotConfigured => "MetricsNotConfigured", + } + } + + /// The condition status: `True`, `False`, or `Unknown`. + #[must_use] + pub const fn status(self) -> &'static str { + match self { + Self::Ready => "True", + Self::MetricsNotConfigured => "Unknown", + Self::NoReadyEndpoints | Self::MetricsStale | Self::MetricsUnreachable | Self::ProviderUnavailable => { + "False" + }, + } + } + + /// The one-word status shown in the STATUS column: `Ready`, `NotReady`, or `Unknown`. + #[must_use] + pub const fn display(self) -> &'static str { + match self { + Self::Ready => "Ready", + Self::MetricsNotConfigured => "Unknown", + Self::NoReadyEndpoints | Self::MetricsStale | Self::MetricsUnreachable | Self::ProviderUnavailable => { + "NotReady" + }, + } + } + + /// Whether the provider is known not to serve: unknown counts as ready. + #[must_use] + pub const fn excludes(self) -> bool { + matches!( + self, + Self::NoReadyEndpoints | Self::MetricsStale | Self::MetricsUnreachable | Self::ProviderUnavailable + ) + } +} + +/// What the signals loop last saw from one provider. +#[derive(Clone, Debug, Default)] +struct Probe { + /// When the first scrape was attempted, which starts the grace before any success. + first_attempt: Option, + /// When the last scrape succeeded. + last_good: Option, + /// Ready endpoints in that scrape, `None` when it exposed no count. + ready_endpoints: Option, + /// Signals from the latest success, until the signals loop publishes them once. + unpublished: Option>, + /// Whether the latest attempt failed. + failing: bool, + /// Consecutive successful scrapes that read zero ready endpoints. + zero_streak: u32, + /// Consecutive successful scrapes that read some ready endpoints. + ready_streak: u32, + /// Whether the count marks the provider not ready, with [`STREAK`] hysteresis both ways. + no_endpoints: bool, +} + +/// One provider's verdict and the detail behind it. +#[derive(Clone, Debug, PartialEq)] +pub struct Verdict { + /// Status and reason. + pub reason: Reason, + /// Human detail for the condition message. + pub message: String, +} + +/// The latest scrape per provider, keyed by network and routing identity. +#[derive(Debug, Default)] +pub struct ReadinessStore(Mutex>); + +/// The store key for `identity` in `network`: identities are unique per network only. +pub(crate) fn key(network: &str, identity: &str) -> String { + format!("{network}/{identity}") +} + +impl ReadinessStore { + /// The probes, recovered if a panicking holder poisoned the lock: each write is whole. + fn probes(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.0.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Record a successful scrape at `now`. + #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] + pub(crate) fn record_success( + &self, + key: &str, + ready_endpoints: Option, + observations: Vec, + now: Instant, + ) { + let mut probes = self.probes(); + let probe = probes.entry(key.to_owned()).or_default(); + probe.first_attempt.get_or_insert(now); + match ready_endpoints { + Some(count) if count < 1.0 => { + probe.zero_streak = probe.zero_streak.saturating_add(1); + probe.ready_streak = 0; + probe.no_endpoints |= probe.zero_streak >= STREAK; + }, + Some(_) => { + probe.ready_streak = probe.ready_streak.saturating_add(1); + probe.zero_streak = 0; + probe.no_endpoints &= probe.ready_streak < STREAK; + }, + // No count to read: reachable is all that is known. + None => { + probe.zero_streak = 0; + probe.ready_streak = 0; + probe.no_endpoints = false; + }, + } + probe.last_good = Some(now); + probe.ready_endpoints = ready_endpoints; + probe.unpublished = Some(observations); + probe.failing = false; + } + + /// Record a failed scrape at `now`, keeping what the last good one saw. + #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] + pub(crate) fn record_failure(&self, key: &str, now: Instant) { + let mut probes = self.probes(); + let probe = probes.entry(key.to_owned()).or_default(); + probe.first_attempt.get_or_insert(now); + probe.failing = true; + } + + /// The verdict for `key` at `now`. + /// + /// `None` until the provider has been attempted, and through the first + /// `stale_after` while no scrape has succeeded yet, so a starting operator + /// publishes nothing rather than a guess. + #[expect( + clippy::significant_drop_tightening, + reason = "the guard covers one small evaluation" + )] + pub(crate) fn verdict(&self, key: &str, unavailable: bool, stale_after: Duration, now: Instant) -> Option { + let probes = self.probes(); + let probe = probes.get(key)?; + let within = |at: Instant| now.saturating_duration_since(at) <= stale_after; + if probe.last_good.is_none() && !unavailable && probe.first_attempt.is_some_and(within) { + return None; + } + Some(evaluate( + probe, + probe.last_good.is_some_and(within), + unavailable, + stale_after, + )) + } + + /// The signals of the latest successful scrape, once: stale load is never republished. + pub(crate) fn take_fresh(&self, key: &str) -> Option> { + self.probes().get_mut(key)?.unpublished.take() + } + + /// Forget providers not in `keep`. + pub(crate) fn retain(&self, keep: &std::collections::BTreeSet) { + self.probes().retain(|key, _| keep.contains(key)); + } +} + +/// The verdict for one probe. `fresh` says whether its last success is within `stale_after`. +fn evaluate(probe: &Probe, fresh: bool, unavailable: bool, stale_after: Duration) -> Verdict { + if unavailable { + return Verdict { + reason: Reason::ProviderUnavailable, + message: "provider is Unavailable".to_owned(), + }; + } + if !fresh { + let reason = if probe.failing { + Reason::MetricsUnreachable + } else { + Reason::MetricsStale + }; + return Verdict { + reason, + message: format!("no successful metrics scrape in the last {}s", stale_after.as_secs()), + }; + } + from_ready_endpoints(probe) +} + +/// The verdict for a freshly scraped provider, from its ready-endpoint count. +fn from_ready_endpoints(probe: &Probe) -> Verdict { + let count = probe.ready_endpoints; + if probe.no_endpoints { + let message = match count { + Some(count) if count >= 1.0 => format!("{count} ready endpoints, awaiting a second scrape"), + _ => format!("0 ready endpoints for {STREAK} or more scrapes"), + }; + return Verdict { + reason: Reason::NoReadyEndpoints, + message, + }; + } + let message = match count { + Some(count) if count < 1.0 => "0 ready endpoints in the latest scrape only".to_owned(), + Some(count) => format!("{count} ready endpoints"), + None => "metrics reachable, no ready-endpoint count exposed".to_owned(), + }; + Verdict { + reason: Reason::Ready, + message, + } +} + +/// The condition type this module owns. +pub const READY_CONDITION: &str = "Ready"; + +/// The `Ready` condition to write, or `None` when `current` already says the same. +/// +/// A changed message alone is not written, so a moving endpoint count does not +/// churn status. A new generation is. `lastTransitionTime` moves only when the status does. +pub(crate) fn ready_condition( + current: &[Condition], + verdict: &Verdict, + now_rfc3339: &str, + generation: Option, +) -> Option { + let held = current.iter().find(|c| c.type_ == READY_CONDITION); + let status = verdict.reason.status(); + if held.is_some_and(|c| { + c.status == status && c.reason == verdict.reason.as_str() && c.observed_generation == generation + }) { + return None; + } + let last_transition_time = held + .filter(|c| c.status == status) + .map_or_else(|| now_rfc3339.to_owned(), |c| c.last_transition_time.clone()); + Some(Condition { + type_: READY_CONDITION.to_owned(), + status: status.to_owned(), + reason: verdict.reason.as_str().to_owned(), + message: verdict.message.clone(), + last_transition_time, + observed_generation: generation, + }) +} + +/// Ready endpoints for `pool` in `observations`, from the first of `names` present. +/// +/// With a pool, only series whose `name` label is that pool count, so an EPP that +/// serves several pools is read for the configured one. +pub(crate) fn ready_endpoints(observations: &[Observation], names: &[&str], pool: Option<&str>) -> Option { + names.iter().find_map(|name| { + observations + .iter() + .filter(|o| o.metric == *name) + .filter(|o| pool.is_none_or(|pool| o.labels.get("name").is_some_and(|n| n == pool))) + .map(|o| o.value) + .reduce(f64::max) + }) +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::too_many_lines, + clippy::shadow_unrelated, + reason = "tests" +)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + const STALE: Duration = Duration::from_secs(15); + + fn sample(metric: &str, pool: &str, value: f64) -> Observation { + Observation { + metric: metric.to_owned(), + labels: BTreeMap::from([("name".to_owned(), pool.to_owned())]), + value, + timestamp_ms: None, + } + } + + #[test] + fn ready_endpoints_prefers_the_current_name_and_reads_the_configured_pool() { + let observations = [ + sample("inference_pool_ready_pods", "qwen3", 4.0), + sample("llm_d_epp_ready_endpoints", "other", 9.0), + sample("llm_d_epp_ready_endpoints", "qwen3", 2.0), + ]; + assert_eq!( + ready_endpoints(&observations, &DEFAULT_READY_ENDPOINTS, Some("qwen3")), + Some(2.0) + ); + assert_eq!( + ready_endpoints(&observations[..1], &DEFAULT_READY_ENDPOINTS, Some("qwen3")), + Some(4.0), + "falls back to the deprecated name" + ); + assert_eq!( + ready_endpoints(&observations, &DEFAULT_READY_ENDPOINTS, Some("absent")), + None + ); + } + + fn reason(store: &ReadinessStore, now: Instant) -> Option { + store.verdict("p", false, STALE, now).map(|verdict| verdict.reason) + } + + #[test] + fn verdict_follows_the_latest_scrape_and_staleness() { + let store = ReadinessStore::default(); + let start = Instant::now(); + assert_eq!(reason(&store, start), None, "never attempted"); + + store.record_success("p", Some(2.0), vec![sample("q", "p", 1.0)], start); + assert_eq!(reason(&store, start), Some(Reason::Ready)); + assert_eq!(store.take_fresh("p").map(|held| held.len()), Some(1)); + assert_eq!(store.take_fresh("p"), None, "fresh signals are published once"); + + store.record_success("p", None, Vec::new(), start); + assert_eq!( + reason(&store, start), + Some(Reason::Ready), + "a scrape without the count is not read as zero" + ); + assert_eq!(store.take_fresh("p").map(|held| held.len()), Some(0)); + + store.record_failure("p", start + STALE); + assert_eq!( + reason(&store, start + STALE), + Some(Reason::Ready), + "a failure within the window keeps the last verdict" + ); + assert_eq!(store.take_fresh("p"), None, "a failure publishes no held load"); + assert_eq!( + reason(&store, start + STALE + Duration::from_secs(1)), + Some(Reason::MetricsUnreachable) + ); + } + + #[test] + fn zero_endpoints_take_two_scrapes_to_exclude_and_two_to_readmit() { + let store = ReadinessStore::default(); + let now = Instant::now(); + let scrape = |count| { + store.record_success("p", Some(count), Vec::new(), now); + reason(&store, now) + }; + assert_eq!(scrape(0.0), Some(Reason::Ready), "one zero reading is not enough"); + assert_eq!(scrape(0.0), Some(Reason::NoReadyEndpoints)); + assert_eq!( + scrape(1.0), + Some(Reason::NoReadyEndpoints), + "one ready reading is not enough" + ); + assert_eq!( + scrape(0.0), + Some(Reason::NoReadyEndpoints), + "a relapse restarts the count" + ); + assert_eq!(scrape(1.0), Some(Reason::NoReadyEndpoints)); + assert_eq!(scrape(1.0), Some(Reason::Ready)); + } + + #[test] + fn a_first_scrape_that_fails_gets_the_staleness_window_before_a_verdict() { + let store = ReadinessStore::default(); + let start = Instant::now(); + store.record_failure("p", start); + assert_eq!(reason(&store, start + STALE), None, "grace, not a guess"); + assert_eq!( + reason(&store, start + STALE + Duration::from_secs(1)), + Some(Reason::MetricsUnreachable) + ); + } + + #[test] + fn providers_are_keyed_per_network() { + let store = ReadinessStore::default(); + let now = Instant::now(); + for _ in 0..STREAK { + store.record_success(&key("east", "p"), Some(0.0), Vec::new(), now); + } + store.record_success(&key("west", "p"), Some(3.0), Vec::new(), now); + let reason_in = |network| { + store + .verdict(&key(network, "p"), false, STALE, now) + .map(|verdict| verdict.reason) + }; + assert_eq!(reason_in("east"), Some(Reason::NoReadyEndpoints)); + assert_eq!(reason_in("west"), Some(Reason::Ready)); + } + + #[test] + fn an_unavailable_provider_is_not_ready_whatever_its_metrics_say() { + let store = ReadinessStore::default(); + let now = Instant::now(); + store.record_success("p", Some(3.0), Vec::new(), now); + let verdict = store.verdict("p", true, STALE, now).unwrap(); + assert_eq!(verdict.reason, Reason::ProviderUnavailable); + assert!(verdict.reason.excludes()); + } + + #[test] + fn the_status_column_reads_like_a_node() { + assert_eq!(Reason::Ready.display(), "Ready"); + assert_eq!(Reason::MetricsNotConfigured.display(), "Unknown"); + for reason in [ + Reason::NoReadyEndpoints, + Reason::MetricsStale, + Reason::MetricsUnreachable, + Reason::ProviderUnavailable, + ] { + assert_eq!(reason.display(), "NotReady", "{reason:?}"); + } + } + + #[test] + fn the_condition_is_written_on_a_status_or_reason_change_only() { + let ready = Verdict { + reason: Reason::Ready, + message: "2 ready endpoints".to_owned(), + }; + let first = ready_condition(&[], &ready, "t0", Some(3)).expect("absent is written"); + assert_eq!( + (first.status.as_str(), first.last_transition_time.as_str()), + ("True", "t0") + ); + + let recount = Verdict { + message: "3 ready endpoints".to_owned(), + ..ready + }; + assert!( + ready_condition(std::slice::from_ref(&first), &recount, "t1", Some(3)).is_none(), + "a new count alone is not news" + ); + let regenerated = ready_condition(std::slice::from_ref(&first), &recount, "t1", Some(4)) + .expect("a new generation is written"); + assert_eq!( + regenerated.last_transition_time, "t0", + "the same status keeps its transition time" + ); + + let down = Verdict { + reason: Reason::NoReadyEndpoints, + message: "0 ready endpoints".to_owned(), + }; + let second = ready_condition(std::slice::from_ref(&first), &down, "t2", Some(3)).unwrap(); + assert_eq!( + (second.status.as_str(), second.last_transition_time.as_str()), + ("False", "t2") + ); + + let unreachable = Verdict { + reason: Reason::MetricsUnreachable, + message: "no scrape".to_owned(), + }; + let third = ready_condition(std::slice::from_ref(&second), &unreachable, "t3", Some(3)).unwrap(); + assert_eq!( + third.last_transition_time, "t2", + "a new reason under the same status keeps the transition time" + ); + } + + #[test] + fn statuses_match_the_condition_contract() { + assert_eq!(Reason::Ready.status(), "True"); + assert_eq!(Reason::MetricsNotConfigured.status(), "Unknown"); + assert!(!Reason::MetricsNotConfigured.excludes(), "unknown is not down"); + for reason in [ + Reason::NoReadyEndpoints, + Reason::MetricsStale, + Reason::MetricsUnreachable, + Reason::ProviderUnavailable, + ] { + assert_eq!(reason.status(), "False"); + assert!(reason.excludes()); + } + } +} diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index c4012f46..dcf24c1f 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -603,6 +603,7 @@ mod tests { network: "test-net".to_owned(), local_site: "site-a".to_owned(), candidates, + excluded: Vec::new(), selection_policy: None, generated_at: None, } @@ -1008,6 +1009,7 @@ mod tests { network: "n".to_owned(), local_site: String::new(), candidates: vec![], + excluded: Vec::new(), selection_policy: None, generated_at: None, }; diff --git a/operator/src/resources/overlay_bridge.rs b/operator/src/resources/overlay_bridge.rs index c79849c2..d1c1ff1d 100644 --- a/operator/src/resources/overlay_bridge.rs +++ b/operator/src/resources/overlay_bridge.rs @@ -118,6 +118,7 @@ mod tests { capacity_weight: 1, }) .collect(), + excluded: Vec::new(), selection_policy: None, generated_at: None, } diff --git a/operator/src/resources/overlay_envelope.rs b/operator/src/resources/overlay_envelope.rs index eb4e9aca..85b78e76 100644 --- a/operator/src/resources/overlay_envelope.rs +++ b/operator/src/resources/overlay_envelope.rs @@ -300,6 +300,7 @@ mod tests { RoutingOverlay { network: "test-net".to_owned(), local_site: "site-a".to_owned(), + excluded: Vec::new(), candidates: vec![RoutingCandidate { kind: "inference_model".to_owned(), name: "model-a".to_owned(), @@ -330,6 +331,7 @@ mod tests { RoutingOverlay { network: "glb-demo".to_owned(), local_site: "east-edge".to_owned(), + excluded: Vec::new(), candidates: vec![ RoutingCandidate { kind: "inference_model".to_owned(), diff --git a/operator/src/resources/provider_metrics.rs b/operator/src/resources/provider_metrics.rs index 705792a6..cf9e96e9 100644 --- a/operator/src/resources/provider_metrics.rs +++ b/operator/src/resources/provider_metrics.rs @@ -107,41 +107,71 @@ pub(crate) struct CollectedMetrics { // Signals collection (poll mode) // --------------------------------------------------------------------------- -/// Scrape each provider's endpoint and keep only its configured coarse signals. +/// Scrape each provider's endpoint, recording its coarse signals and readiness. /// /// Sibling of [`collect_provider_metrics_with_refresh_interval`], which parses /// the same text into [`scoring::BackendMetrics`] for local scoring. This keeps /// the provider's own exposition narrowed to its declared `signalNames`, so the /// wire carries a coarse rollup rather than the full `/metrics` firehose. Fails /// closed on TLS: a provider whose TLS will not resolve is skipped, never -/// scraped in plaintext. A failed scrape leaves the last value to expire. +/// scraped in plaintext, and counts as a failed scrape. pub(crate) async fn collect_provider_signals( network_name: &str, providers: &[InferenceProvider], client: Option<&kube::Client>, -) -> HashMap> { - let mut out = HashMap::new(); + readiness: &crate::readiness::ReadinessStore, +) { for provider in providers { if provider.spec.grid_network_ref != network_name { continue; } - if let Some((identity, observations)) = scrape_provider_signals(provider, client).await { - out.insert(identity, observations); + let Some(plan) = signal_scrape_plan(provider) else { + continue; + }; + let key = crate::readiness::key(network_name, plan.identity); + match scrape_provider_signals(provider, &plan, client).await { + Some(text) => record_scrape(readiness, &key, provider, &plan, &text), + None => readiness.record_failure(&key, Instant::now()), } } - out } -/// Scrape one provider's coarse signals, or `None` to leave its last value be. -/// -/// Every skip and failure returns `None`: a provider absent from the collection -/// is left alone rather than erased, so a missed scrape expires on its own. +/// Record one successful scrape: its ready-endpoint count and its declared signals. +fn record_scrape( + readiness: &crate::readiness::ReadinessStore, + key: &str, + provider: &InferenceProvider, + plan: &SignalScrapePlan<'_>, + text: &str, +) { + let parsed = crate::signals::parse(text); + let pool = provider + .spec + .metrics_config + .as_ref() + .and_then(|mc| mc.pool_name.as_deref()); + let ready = crate::readiness::ready_endpoints(&parsed, plan.ready_names(), pool); + let observations = parsed + .into_iter() + .filter(|o| plan.wanted.contains(o.metric.as_str())) + // A local sample's freshness is its collection time, so drop any trailing + // timestamp. Only relayed peer samples carry a per-sample stamp. + .map(|mut o| { + o.timestamp_ms = None; + o + }) + .collect(); + readiness.record_success(key, ready, observations, Instant::now()); +} + +/// Scrape one provider's exposition, or `None` when TLS or the scrape fails. async fn scrape_provider_signals( provider: &InferenceProvider, + plan: &SignalScrapePlan<'_>, client: Option<&kube::Client>, -) -> Option<(String, Vec)> { +) -> Option { let mc = provider.spec.metrics_config.as_ref()?; - let (identity, url, wanted) = signal_scrape_plan(provider)?; + let identity = plan.identity; let tls_config = match resolve_tls_config(mc.tls.as_ref(), client, identity).await { Ok(cfg) => cfg, Err((_reason, e)) => { @@ -152,43 +182,54 @@ async fn scrape_provider_signals( }, }; let timeout = parse_metrics_timeout(&mc.timeout); - let text = scrape_metrics(&url, timeout, tls_config, mc.auth.as_ref().zip(client)) + scrape_metrics(&plan.url, timeout, tls_config, mc.auth.as_ref().zip(client)) .await .inspect_err(|e| { - tracing::debug!(provider = identity, error = %e, "signals: provider scrape failed; last value left to expire"); - }) - .ok()?; - let observations = crate::signals::parse(&text) - .into_iter() - .filter(|o| wanted.contains(o.metric.as_str())) - // A local sample's freshness is its collection time, so drop any trailing - // timestamp. Only relayed peer samples carry a per-sample stamp. - .map(|mut o| { - o.timestamp_ms = None; - o + tracing::debug!(provider = identity, error = %e, "signals: provider scrape failed"); }) - .collect(); - Some((identity.to_owned(), observations)) + .ok() } -/// The scrape target for a provider's coarse signals, if it is eligible. +/// What to scrape for one provider and which of its series to republish. +pub(crate) struct SignalScrapePlan<'provider> { + /// The provider's routing identity. + pub(crate) identity: &'provider str, + /// The metrics URL. + pub(crate) url: String, + /// Declared signal names republished as coarse signals; may be empty. + pub(crate) wanted: std::collections::BTreeSet, + /// A declared ready-endpoints metric, replacing the defaults. + ready_override: Option<&'provider str>, +} + +impl SignalScrapePlan<'_> { + /// The ready-endpoints names to read, preferred first. + fn ready_names(&self) -> &[&str] { + self.ready_override + .as_ref() + .map_or(&crate::readiness::DEFAULT_READY_ENDPOINTS[..], std::slice::from_ref) + } +} + +/// The scrape plan for a provider, if it is eligible. /// /// Pure and synchronous: eligibility is decided here so the scrape path stays /// the I/O alone. `None` for a provider with no metrics config, no routing -/// identity, a blank endpoint, or no declared signal names. -fn signal_scrape_plan(provider: &InferenceProvider) -> Option<(&str, String, std::collections::BTreeSet)> { +/// identity, or a blank endpoint. A provider with no signal names is still +/// scraped, for readiness. +pub(crate) fn signal_scrape_plan(provider: &InferenceProvider) -> Option> { let mc = provider.spec.metrics_config.as_ref()?; let identity = routing_identity(provider)?; let endpoint = provider.spec.endpoint.trim(); if endpoint.is_empty() || mc.metrics_endpoint.as_deref().is_some_and(|ep| ep.trim().is_empty()) { return None; } - let wanted = signal_metric_names(&mc.signal_names); - if wanted.is_empty() { - return None; - } - let url = metrics_url(mc.metrics_endpoint.as_deref().unwrap_or(endpoint), &mc.path); - Some((identity, url, wanted)) + Some(SignalScrapePlan { + identity, + url: metrics_url(mc.metrics_endpoint.as_deref().unwrap_or(endpoint), &mc.path), + wanted: signal_metric_names(&mc.signal_names), + ready_override: mc.signal_names.ready_endpoints.as_deref(), + }) } /// The source metric names a provider declares for its coarse signals. @@ -874,6 +915,7 @@ mod tests { prefix_cache_hit_ratio: Some("my_prefix".to_owned()), error_rate: Some("my_errors".to_owned()), healthy: Some("my_health".to_owned()), + ready_endpoints: None, }; let names = metric_names_from_config(&cfg, None, None); assert_eq!(names.queue_depth.as_deref(), Some("my_queue")); diff --git a/operator/src/resources/routing_overlay.rs b/operator/src/resources/routing_overlay.rs index e1468dcf..1f357c49 100644 --- a/operator/src/resources/routing_overlay.rs +++ b/operator/src/resources/routing_overlay.rs @@ -931,6 +931,11 @@ pub struct RoutingOverlay { /// freshness, score, then alphabetical tiebreak. pub candidates: Vec, + /// Candidates excluded from routing, kept for the serving config so the gateway + /// can answer 503 for a known model. Never on the overlay's own wire. + #[serde(skip)] + pub excluded: Vec, + /// Optional explicit local selection policy for Praxis. An absent field is /// intentionally backward-compatible and means deterministic selection. #[serde(skip_serializing_if = "Option::is_none")] @@ -1304,7 +1309,9 @@ pub fn render_routing_overlay_with_admission( } enrich_candidates(&mut candidates, local_site, sites, network_name, &admission_map); - candidates.retain(|c| c.admission_state != Some(AdmissionState::Excluded)); + let (excluded, mut candidates): (Vec<_>, Vec<_>) = candidates + .into_iter() + .partition(|c| c.admission_state == Some(AdmissionState::Excluded)); let policy = network .spec @@ -1381,6 +1388,7 @@ pub fn render_routing_overlay_with_admission( network: network_name.to_owned(), local_site: local_site.to_owned(), candidates, + excluded, selection_policy, generated_at: generated_at.map(str::to_owned), }) diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index 64830931..a90d3e8b 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -81,6 +81,16 @@ pub(crate) struct ServingCandidate { pub(crate) cluster: String, /// Freshness carried from the overlay. pub(crate) fresh: bool, + /// Whether it takes new requests. Omitted when it does, so such a config + /// still loads on a gateway that predates the field. + #[serde(skip_serializing_if = "admits_new")] + pub(crate) admission: AdmissionState, +} + +/// Whether `admission` is the default, which the gateway assumes when absent. +#[expect(clippy::trivially_copy_pass_by_ref, reason = "serde skip_serializing_if signature")] +fn admits_new(admission: &AdmissionState) -> bool { + *admission == AdmissionState::NewAndExisting } /// One peer signals endpoint and the local identity material to reach it. @@ -133,7 +143,7 @@ pub(crate) fn render<'member, Members>( where Members: IntoIterator, { - let candidates = candidates(&overlay.candidates, &overlay.local_site); + let candidates = candidates(overlay); if candidates.is_empty() { return None; } @@ -188,48 +198,70 @@ pub(crate) fn dialable_members<'snap>( .collect() } -/// Admitted inference candidates within gateway limits, deduplicated and ordered. -fn candidates(overlay: &[RoutingCandidate], local_site: &str) -> Vec { - // Local first so cold start, before any signal, prefers this site. - let mut unique: BTreeMap<(bool, &str, &str, &str), bool> = BTreeMap::new(); - for candidate in overlay.iter().filter(|candidate| routable(candidate)) { +/// A candidate's identity: remote after local, then site, name, cluster. +type CandidateKey<'overlay> = (bool, &'overlay str, &'overlay str, &'overlay str); + +/// Each routable candidate once, with its freshness and admission. +/// +/// Any stale or more restricted duplicate marks the tuple, whatever the order. +fn dedup<'overlay>( + overlay: impl Iterator, + local_site: &str, +) -> BTreeMap, (bool, AdmissionState)> { + let mut unique = BTreeMap::new(); + for candidate in overlay.filter(|candidate| routable(candidate)) { let key = ( candidate.site != local_site, candidate.site.as_str(), candidate.name.as_str(), candidate.cluster.as_str(), ); - // Any stale duplicate marks the tuple stale, whatever the order. + let admission = candidate.admission_state.unwrap_or(AdmissionState::NewAndExisting); unique .entry(key) - .and_modify(|fresh| *fresh &= candidate.fresh) - .or_insert(candidate.fresh); + .and_modify(|(fresh, held): &mut (bool, AdmissionState)| { + *fresh &= candidate.fresh; + *held = (*held).max(admission); + }) + .or_insert((candidate.fresh, admission)); } - if unique.len() > MAX_CANDIDATES { + unique +} + +/// Inference candidates within gateway limits, deduplicated and ordered. +/// +/// Past the cap, candidates taking new requests are kept before the rest. +fn candidates(overlay: &RoutingOverlay) -> Vec { + // Local first so cold start, before any signal, prefers this site. + let unique = dedup(overlay.candidates.iter().chain(&overlay.excluded), &overlay.local_site); + let mut kept: Vec<_> = unique.into_iter().collect(); + if kept.len() > MAX_CANDIDATES { tracing::warn!( - candidates = unique.len(), + candidates = kept.len(), "serving config: dropping candidates past the gateway cap" ); + kept.sort_by_key(|(key, (_, admission))| (*admission != AdmissionState::NewAndExisting, *key)); + kept.truncate(MAX_CANDIDATES); + kept.sort_by_key(|(key, _)| *key); } - unique - .into_iter() - .take(MAX_CANDIDATES) - .map(|((_, site, name, cluster), fresh)| ServingCandidate { + kept.into_iter() + .map(|((_, site, name, cluster), (fresh, admission))| ServingCandidate { kind: INFERENCE_MODEL, name: name.to_owned(), site: site.to_owned(), cluster: cluster.to_owned(), fresh, + admission, }) .collect() } -/// An inference candidate admitting new requests that the gateway will accept. +/// An inference candidate the gateway will accept, whatever its admission. +/// +/// A candidate not taking new requests stays in, so the gateway can tell a known +/// down model, answered with 503, from an unknown one. fn routable(candidate: &RoutingCandidate) -> bool { candidate.kind == INFERENCE_MODEL - && candidate - .admission_state - .is_none_or(|state| state == AdmissionState::NewAndExisting) && [&candidate.name, &candidate.site, &candidate.cluster] .into_iter() .all(|id| valid_id(id)) @@ -403,6 +435,7 @@ mod tests { network: "grid".to_owned(), local_site: "site-a".to_owned(), candidates, + excluded: Vec::new(), selection_policy: None, generated_at: None, } @@ -453,13 +486,8 @@ mod tests { } #[test] - fn candidates_the_gateway_would_reject_or_never_admit_are_dropped() { + fn candidates_the_gateway_would_reject_are_dropped() { let cases = [ - ("excluded", cand("llama", "site-b", "pool-b", Some("none"))), - ( - "existing only", - cand("llama", "site-b", "pool-b", Some("existing_only")), - ), ("blank cluster", cand("llama", "site-b", " ", None)), ( "oversized name", @@ -474,6 +502,68 @@ mod tests { assert!(render(&overlay(vec![mcp]), [], &INPUTS).is_none(), "mcp_tool"); } + #[test] + fn an_excluded_candidate_reaches_the_serving_config_but_not_the_overlay_wire() { + let mut source = overlay(vec![cand("llama", "site-a", "pool-a", None)]); + source.excluded = vec![cand("llama", "site-b", "pool-b", Some("none"))]; + let rendered = render(&source, [], &INPUTS).expect("rendered"); + let config: serde_json::Value = serde_json::from_str(&to_text(&rendered).expect("text")).expect("json"); + let pool_b = config["candidates"] + .as_array() + .expect("candidates") + .iter() + .find(|c| c["cluster"] == "pool-b") + .expect("the excluded candidate is listed"); + assert_eq!(pool_b["admission"], "none"); + let wire = serde_json::to_string(&source).expect("overlay json"); + assert!(!wire.contains("pool-b"), "the overlay wire is unchanged: {wire}"); + } + + #[test] + fn past_the_cap_candidates_taking_new_requests_are_kept_first() { + let excluded: Vec<_> = (0..MAX_CANDIDATES) + .map(|i| cand("llama", "site-a", &format!("down-{i:04}"), Some("none"))) + .collect(); + let mut source = overlay(vec![cand("llama", "site-z", "up", None)]); + source.excluded = excluded; + let rendered = render(&source, [], &INPUTS).expect("rendered"); + assert_eq!(rendered.candidates.len(), MAX_CANDIDATES); + assert!( + rendered.candidates.iter().any(|c| c.cluster == "up"), + "the one admitted candidate survives the cap" + ); + } + + #[test] + fn a_candidate_not_taking_new_requests_stays_in_with_its_admission() { + let rendered = render( + &overlay(vec![ + cand("llama", "site-a", "pool-a", None), + cand("llama", "site-b", "pool-b", Some("none")), + cand("llama", "site-d", "pool-d", Some("existing_only")), + ]), + [], + &INPUTS, + ) + .expect("rendered"); + let config: serde_json::Value = serde_json::from_str(&to_text(&rendered).expect("text")).expect("json"); + let admission: Vec<(&str, Option<&str>)> = config["candidates"] + .as_array() + .expect("candidates") + .iter() + .map(|c| (c["site"].as_str().unwrap_or(""), c["admission"].as_str())) + .collect(); + assert_eq!( + admission, + [ + ("site-a", None), + ("site-b", Some("none")), + ("site-d", Some("existing_only")) + ], + "the default is omitted, so an older gateway still loads an all-admitted config" + ); + } + #[test] fn unsafe_or_irrelevant_peers_are_refused() { let cases = [ From 74f776525d32ccdd86e933d3b86215006c3a4e19 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 14:07:21 -0400 Subject: [PATCH 03/10] feat(operator)!: publish a provider's in-flight requests, capacity, and saturation grid_provider_in_flight_requests is the larger of the EPP's per-endpoint count and the pool averages times ready endpoints, plus flow control's queue. spec.maxRunning now counts one endpoint; grid_provider_capacity_requests is it times fresh ready endpoints, and grid_provider_saturation_ratio is in-flight over capacity for a finite non-negative count. The grid-operator chart passes grid.providers..maxRunning through. Series take the signal contract names. Reinstall with per-endpoint maxRunning. Signed-off-by: Sam Batschelet --- charts/grid-operator/README.md | 1 + .../templates/crds/inferenceprovider.yaml | 11 + .../templates/inferenceprovider.yaml | 3 + charts/grid-operator/tests/grid_test.yaml | 26 ++ charts/grid-operator/values.yaml | 3 + deploy/crds/inferenceprovider.yaml | 11 + docs/architecture/crds.md | 7 + docs/architecture/polling-metrics.md | 30 +++ .../testdata/serving-config.json | 2 +- operator/src/controller/grid_network.rs | 113 ++++++++- operator/src/controller/inference_provider.rs | 1 + operator/src/crd/inference_provider.rs | 9 + operator/src/readiness.rs | 238 ++++++++++++++++++ operator/src/resources/provider_metrics.rs | 24 ++ operator/src/resources/serving_config.rs | 10 +- 15 files changed, 475 insertions(+), 14 deletions(-) diff --git a/charts/grid-operator/README.md b/charts/grid-operator/README.md index e5cea1a3..1a5be7d5 100644 --- a/charts/grid-operator/README.md +++ b/charts/grid-operator/README.md @@ -156,6 +156,7 @@ RELEASE=grid-operator; NAMESPACE=grid-system; for crd in agenttoolproviders grid |-----|------|---------|-------------| | `crds.enabled` | bool | `true` | Install and upgrade the Grid CRDs. `false` when a platform owns them. | | `crds.keep` | bool | `true` | Keep the CRDs on `helm uninstall` and an Argo CD delete or prune. | +| `grid.providers` | object | `{}` | InferenceProviders this site serves, keyed by name. `model` defaults to the name, `providerKind` to `vllm`, `backendKind` to `local_model`. `maxRunning` is the most requests one endpoint runs at once (vLLM max-num-seqs); without it the provider's capacity, and so its saturation, is unknown. | | `rbac.enrollmentNamespace` | string | `""` | The grid-enrollment namespace. The render fails if the operator would get Secret access there. | | `rbac.metricsScraper` | bool | `true` | Create the metrics scraper ServiceAccount, allowed only GET on the nonResourceURL /metrics, and let the operator mint short-lived tokens for it. An llm-d EPP serving bearer-authenticated metrics (the default) admits a scrape with that token (metricsConfig.auth type serviceAccountToken). The operator never sends its own token. | | `replicaCount` | int | `1` | Operator replicas. Must be 1 (schema-enforced). | diff --git a/charts/grid-operator/templates/crds/inferenceprovider.yaml b/charts/grid-operator/templates/crds/inferenceprovider.yaml index 613adc04..eeb1216e 100644 --- a/charts/grid-operator/templates/crds/inferenceprovider.yaml +++ b/charts/grid-operator/templates/crds/inferenceprovider.yaml @@ -300,6 +300,17 @@ spec: - message: set exactly one of caSecretRef and caConfigMapRef rule: has(self.caSecretRef) != has(self.caConfigMapRef) type: object + maxRunning: + description: |- + Most requests one endpoint runs at once: vLLM max-num-seqs. + + The site's operator multiplies it by the EPP's fresh ready endpoints and publishes + the product as `grid_provider_capacity_requests`, the capacity gateways weigh the + provider's load against. When absent, or no endpoint is fresh, capacity is unknown. + format: uint32 + minimum: 1.0 + nullable: true + type: integer metricsConfig: description: |- Prometheus metrics scraping configuration. diff --git a/charts/grid-operator/templates/inferenceprovider.yaml b/charts/grid-operator/templates/inferenceprovider.yaml index 576efd32..d2ed256c 100644 --- a/charts/grid-operator/templates/inferenceprovider.yaml +++ b/charts/grid-operator/templates/inferenceprovider.yaml @@ -23,6 +23,9 @@ spec: {{- with .capacityWeight }} capacityWeight: {{ . }} {{- end }} + {{- with .maxRunning }} + maxRunning: {{ . }} + {{- end }} {{- with .routingClusterRef }} routingClusterRef: {{ . | quote }} {{- end }} diff --git a/charts/grid-operator/tests/grid_test.yaml b/charts/grid-operator/tests/grid_test.yaml index 41e1f03b..3d73b8b1 100644 --- a/charts/grid-operator/tests/grid_test.yaml +++ b/charts/grid-operator/tests/grid_test.yaml @@ -135,6 +135,32 @@ tests: - name: Qwen3-Coder-30B-A3B capabilities: [text_generation] + - it: passes maxRunning through to the provider spec + template: templates/inferenceprovider.yaml + set: + site.name: east + grid.id: lab + grid.providers: + qwen3: + endpoint: http://qwen3.llm-d.svc:80 + maxRunning: 64 + asserts: + - equal: + path: spec.maxRunning + value: 64 + + - it: leaves maxRunning unset when a provider omits it + template: templates/inferenceprovider.yaml + set: + site.name: east + grid.id: lab + grid.providers: + qwen3: + endpoint: http://qwen3.llm-d.svc:80 + asserts: + - notExists: + path: spec.maxRunning + - it: keeps YAML-ambiguous names strings templates: - templates/gridnetwork.yaml diff --git a/charts/grid-operator/values.yaml b/charts/grid-operator/values.yaml index 0938d7f2..927f775f 100644 --- a/charts/grid-operator/values.yaml +++ b/charts/grid-operator/values.yaml @@ -28,11 +28,14 @@ grid: swimKeySecretName: grid-swim-key # -- InferenceProviders this site serves, keyed by name, as in the grid-site chart. # model defaults to the name, providerKind to vllm, and backendKind to local_model. + # maxRunning is the most requests one endpoint runs at once (vLLM max-num-seqs); without + # it the provider's capacity, and so its saturation, is unknown. providers: {} # qwen3: # model: Qwen3-Coder-30B-A3B # endpoint: http://qwen3-epp-service.llm-d.svc:80 # routingClusterRef: site-d + # maxRunning: 64 # -- Grid CRDs. crds: diff --git a/deploy/crds/inferenceprovider.yaml b/deploy/crds/inferenceprovider.yaml index 1db5b55a..f6e802a3 100644 --- a/deploy/crds/inferenceprovider.yaml +++ b/deploy/crds/inferenceprovider.yaml @@ -292,6 +292,17 @@ spec: - message: set exactly one of caSecretRef and caConfigMapRef rule: has(self.caSecretRef) != has(self.caConfigMapRef) type: object + maxRunning: + description: |- + Most requests one endpoint runs at once: vLLM max-num-seqs. + + The site's operator multiplies it by the EPP's fresh ready endpoints and publishes + the product as `grid_provider_capacity_requests`, the capacity gateways weigh the + provider's load against. When absent, or no endpoint is fresh, capacity is unknown. + format: uint32 + minimum: 1.0 + nullable: true + type: integer metricsConfig: description: |- Prometheus metrics scraping configuration. diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index b18cb62f..548a492b 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -591,6 +591,13 @@ qwen3-site-a self_hosted Ready 3d qwen3-site-b self_hosted NotReady 3d ``` +`spec.maxRunning` is the most requests one endpoint runs at once, vLLM +max-num-seqs. Its site's operator multiplies it by the EPP's fresh ready endpoints +and publishes the product as `grid_provider_capacity_requests`, the capacity gateways weigh +the provider's load against. When it is absent, or no endpoint is fresh, capacity is +unknown. Before this release `maxRunning` counted the whole pool: reinstall with +per-endpoint values. + `spec.capacityWeight` is an optional positive relative provider capacity from `1` through `1000`, used only with `GridNetwork.spec.selectionPolicy.mode: weightedRandom` and `placementPolicy.strategy: static`. If omitted, the diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index 0c32a566..dc7421a9 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -72,6 +72,36 @@ When every candidate for a model is excluded, the gateway answers 503 with With the defaults of a 5 s scrape and a 5 s poll, exclusion takes at most about 15 s and rejoin about 10 s. +## Provider In-flight + +Each site's operator also publishes how many requests each provider holds, as +`grid_provider_in_flight_requests{grid_site,grid_provider}`. Every input comes from the EPP's +`/metrics`, and nothing scrapes vLLM. The value is the larger of two estimates, plus the +requests the EPP's flow control holds for the pool (`llm_d_epp_flow_control_queue_size`): + +- The EPP's per-endpoint `llm_d_epp_inflight_requests`, summed, taking each endpoint's + largest count across producer instances. It needs the EPP's inflight-load-producer. +- The pool's average running plus average queued requests, times ready endpoints. + +The larger, so an EPP restart that zeroes its count does not make the site look idle. +The per-endpoint count carries no pool label, so when one EPP serves more than one pool the +operator uses the pool averages alone. A site with no fresh endpoint publishes nothing, +since its averages are frozen at their last value, and the gateway reads it as unknown. + +On a prefill/decode pool the EPP counts a request on its prefill and its decode +endpoint, so the value counts endpoint occupancy, up to twice the requests. Capacity +must therefore count the slots of every ready endpoint, prefill and decode alike. + +Point `metricsConfig.metricsEndpoint` at the EPP Service. A pod or headless address +can reach a standby replica, which reports no series. + +Capacity is the provider's per-endpoint `spec.maxRunning` times the EPP's fresh ready +endpoints, published as `grid_provider_capacity_requests`, so it shrinks when pods are lost. +With no fresh endpoint it is unpublished, and the gateway reads it as unknown. + +With both known, the operator also publishes their ratio, requests held over +capacity, as `grid_provider_saturation_ratio`. Gateways choose sites by it. + ## Failure Behavior | Condition | Signal produced | Routing effect | diff --git a/gateway/ai-grid-filters/testdata/serving-config.json b/gateway/ai-grid-filters/testdata/serving-config.json index 570b01ab..ad100ebf 100644 --- a/gateway/ai-grid-filters/testdata/serving-config.json +++ b/gateway/ai-grid-filters/testdata/serving-config.json @@ -1,7 +1,7 @@ { "local_site": "site-a", "window_secs": 60, - "load_window_ms": 30000, + "load_window_ms": 10000, "candidates": [ { "kind": "inference_model", diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 53f550a5..09b1647a 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -499,6 +499,8 @@ fn readiness_entry<'provider>( now: Instant, ) -> Option> { let identity = routing_overlay::routing_identity(provider)?; + let key = readiness::key(&provider.spec.grid_network_ref, identity); + let capacity = capacity_sample(provider, ctx.readiness.ready_endpoints(&key)); if provider_metrics::signal_scrape_plan(provider).is_none() { return Some(ReadinessEntry { identity, @@ -506,6 +508,7 @@ fn readiness_entry<'provider>( reason: readiness::Reason::MetricsNotConfigured, message: "no metricsConfig endpoint to read readiness from".to_owned(), }, + // With no scrape there are no ready endpoints to multiply, so capacity is unknown. published: None, }); } @@ -513,21 +516,62 @@ fn readiness_entry<'provider>( let ready = ready_sample(&verdict); // A fresh scrape republishes with the verdict. Without one, the last published // entry ages on its own, unless the verdict turned not ready. - let fresh = ctx - .readiness - .take_fresh(&readiness::key(&provider.spec.grid_network_ref, identity)); - let published = match fresh { + let fresh = ctx.readiness.take_fresh(&key); + Some(ReadinessEntry { + identity, + published: published_signals(fresh, ready, capacity, verdict.reason.excludes()), + verdict, + }) +} + +/// What to publish for a scraped provider: a fresh scrape with the verdict and capacity, +/// or only those when it is not ready. Without either, the last entry ages on its own. +fn published_signals( + fresh: Option>, + ready: signals::Observation, + capacity: Option, + excluded: bool, +) -> Option> { + match fresh { Some(mut held) => { held.push(ready); + held.extend(saturation_sample(&held, capacity.as_ref())); + held.extend(capacity); Some(held) }, - None if verdict.reason.excludes() => Some(vec![ready]), + None if excluded => Some(std::iter::once(ready).chain(capacity).collect()), None => None, - }; - Some(ReadinessEntry { - identity, - verdict, - published, + } +} + +/// The `grid_provider_saturation_ratio` sample: requests held in `held` over `capacity`, `None` +/// when either is unknown. +fn saturation_sample( + held: &[signals::Observation], + capacity: Option<&signals::Observation>, +) -> Option { + let in_flight = held.iter().find(|o| o.metric == readiness::IN_FLIGHT_SIGNAL)?.value; + let capacity = capacity?.value; + // A non-finite or negative count from a misbehaving EPP publishes no ratio, never inf. + (capacity > 0.0 && in_flight.is_finite() && in_flight >= 0.0).then(|| signals::Observation { + metric: readiness::SATURATION_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value: in_flight / capacity, + timestamp_ms: None, + }) +} + +/// The `grid_provider_capacity_requests` sample: the provider's per-endpoint `maxRunning` times its +/// `ready` endpoints. `None` when it declares none or no endpoint is fresh, which leaves +/// capacity unknown rather than zero. +fn capacity_sample(provider: &InferenceProvider, ready: Option) -> Option { + let max_running = provider.spec.max_running?; + let ready = ready.filter(|ready| *ready > 0.0)?; + Some(signals::Observation { + metric: readiness::CAPACITY_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value: f64::from(max_running) * ready, + timestamp_ms: None, }) } @@ -3995,6 +4039,55 @@ fn parse_metrics_refresh_interval(value: &str) -> Result InferenceProvider { serde_json::from_value(serde_json::json!({ diff --git a/operator/src/controller/inference_provider.rs b/operator/src/controller/inference_provider.rs index 21a8ba52..dd2f6e4d 100644 --- a/operator/src/controller/inference_provider.rs +++ b/operator/src/controller/inference_provider.rs @@ -2688,6 +2688,7 @@ mod tests { ) -> InferenceProviderSpec { InferenceProviderSpec { capacity_weight: None, + max_running: None, grid_network_ref: "net".to_owned(), access_policy: crate::crd::auth::AccessPolicy::default(), auth: None, diff --git a/operator/src/crd/inference_provider.rs b/operator/src/crd/inference_provider.rs index 7f9abf36..e98e5610 100644 --- a/operator/src/crd/inference_provider.rs +++ b/operator/src/crd/inference_provider.rs @@ -65,6 +65,15 @@ pub struct InferenceProviderSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub capacity_weight: Option, + /// Most requests one endpoint runs at once: vLLM max-num-seqs. + /// + /// The site's operator multiplies it by the EPP's fresh ready endpoints and publishes + /// the product as `grid_provider_capacity_requests`, the capacity gateways weigh the + /// provider's load against. When absent, or no endpoint is fresh, capacity is unknown. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_running: Option, + /// Cost information. pub cost: Option, diff --git a/operator/src/readiness.rs b/operator/src/readiness.rs index adcc533f..cee161a6 100644 --- a/operator/src/readiness.rs +++ b/operator/src/readiness.rs @@ -22,9 +22,35 @@ pub(crate) const DEFAULT_READY_ENDPOINTS: [&str; 2] = ["llm_d_epp_ready_endpoint /// pool that just drained can read ready once before its first request lands. pub(crate) const STREAK: u32 = 2; +/// The series this site publishes per provider declaring `maxRunning`: that capacity. +pub const CAPACITY_SIGNAL: &str = "grid_provider_capacity_requests"; + +/// The series this site publishes per provider with both counts: requests held over capacity. +pub const SATURATION_SIGNAL: &str = "grid_provider_saturation_ratio"; + /// The series this site publishes per provider: 1 when ready, 0 when not. pub const READY_SIGNAL: &str = "grid_provider_ready"; +/// The resolved signal for requests a provider has running and queued. +pub const IN_FLIGHT_SIGNAL: &str = "grid_provider_in_flight_requests"; + +/// The EPP's per-endpoint in-flight count, from its inflight-load-producer plugin. It +/// increments after flow control admits a request, once per scheduling profile's target, so +/// on a P/D pool a request counts on its prefill and its decode endpoint. +const EPP_IN_FLIGHT: &str = "llm_d_epp_inflight_requests"; + +/// Requests the EPP's flow control holds before scheduling, labeled by `inference_pool`. +const EPP_FLOW_CONTROL_QUEUE: &str = "llm_d_epp_flow_control_queue_size"; + +/// Pool averages that stand in when the EPP runs no inflight-load-producer. +const EPP_AVERAGES: [[&str; 2]; 2] = [ + [ + "llm_d_epp_average_running_requests", + "inference_pool_average_running_requests", + ], + ["llm_d_epp_average_queue_size", "inference_pool_average_queue_size"], +]; + /// The `Ready` condition's status and reason for one provider. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Reason { @@ -209,6 +235,11 @@ impl ReadinessStore { self.probes().get_mut(key)?.unpublished.take() } + /// Ready endpoints in the provider's last successful scrape, `None` when it exposed no count. + pub(crate) fn ready_endpoints(&self, key: &str) -> Option { + self.probes().get(key)?.ready_endpoints + } + /// Forget providers not in `keep`. pub(crate) fn retain(&self, keep: &std::collections::BTreeSet) { self.probes().retain(|key, _| keep.contains(key)); @@ -309,6 +340,96 @@ pub(crate) fn ready_endpoints(observations: &[Observation], names: &[&str], pool }) } +/// Where a provider's in-flight count came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum InFlightSource { + /// The EPP's inflight-load-producer, at or above the pool averages. + Epp, + /// The pool averages of running and queued times ready endpoints, above the EPP count or + /// in place of it. + EngineAverages, +} + +/// Requests the provider holds: the larger of the EPP's per-endpoint count summed and its +/// pool averages of running and queued times `ready`, plus requests flow control holds. +/// +/// The larger of the two, so an EPP restart that zeroes its count does not read as idle. The +/// per-endpoint count carries no pool label, so it is skipped when the EPP serves several +/// pools. `None` when neither source is present, or the site has no fresh endpoint behind +/// frozen averages. +pub(crate) fn in_flight( + observations: &[Observation], + ready: Option, + pool: Option<&str>, +) -> Option<(f64, InFlightSource)> { + let counted = (!serves_several_pools(observations)) + .then(|| epp_count(observations)) + .flatten(); + // No fresh endpoint means the averages are frozen at their last value: unknown, not idle. + let averaged = ready.filter(|ready| *ready > 0.0).and_then(|ready| { + EPP_AVERAGES + .iter() + .map(|names| ready_endpoints(observations, names, pool)) + .sum::>() + .map(|per_endpoint| per_endpoint * ready) + }); + let (held, source) = match (counted, averaged) { + (Some(counted), Some(averaged)) if averaged > counted => (averaged, InFlightSource::EngineAverages), + (Some(counted), _) => (counted, InFlightSource::Epp), + (None, Some(averaged)) => (averaged, InFlightSource::EngineAverages), + (None, None) => return None, + }; + Some((held + flow_control_queued(observations, pool), source)) +} + +/// The EPP's in-flight count summed over endpoints, each endpoint's largest across producer +/// instances so two producers do not double it. `None` when the EPP exports none. +fn epp_count(observations: &[Observation]) -> Option { + let mut producers: std::collections::BTreeMap<&str, std::collections::BTreeMap<(&str, &str), f64>> = + std::collections::BTreeMap::new(); + for observation in observations.iter().filter(|o| o.metric == EPP_IN_FLIGHT) { + let label = |name: &str| observation.labels.get(name).map_or("", String::as_str); + // Series per fairness and priority add up; producer instances repeat the same requests. + *producers + .entry(label("producer_name")) + .or_default() + .entry((label("namespace"), label("endpoint_name"))) + .or_insert(0.0) += observation.value; + } + let mut peak: std::collections::BTreeMap<(&str, &str), f64> = std::collections::BTreeMap::new(); + for counts in producers.values() { + for (endpoint, count) in counts { + let held = peak.entry(*endpoint).or_insert(0.0); + *held = held.max(*count); + } + } + (!peak.is_empty()).then(|| peak.values().sum()) +} + +/// Whether the EPP reports more than one pool or namespace, so its unlabeled per-endpoint +/// count cannot be attributed to one pool. +fn serves_several_pools(observations: &[Observation]) -> bool { + let distinct = |metrics: &[&str], label: &str| { + observations + .iter() + .filter(|o| metrics.contains(&o.metric.as_str())) + .filter_map(|o| o.labels.get(label)) + .collect::>() + .len() + }; + distinct(&DEFAULT_READY_ENDPOINTS, "name") > 1 || distinct(&[EPP_IN_FLIGHT], "namespace") > 1 +} + +/// Requests the EPP's flow control holds for `pool`, 0 when it exports none. +fn flow_control_queued(observations: &[Observation], pool: Option<&str>) -> f64 { + observations + .iter() + .filter(|o| o.metric == EPP_FLOW_CONTROL_QUEUE) + .filter(|o| pool.is_none_or(|pool| o.labels.get("inference_pool").is_some_and(|p| p == pool))) + .map(|o| o.value) + .sum() +} + #[cfg(test)] #[expect(clippy::allow_attributes, reason = "blanket test suppressions")] #[allow( @@ -532,4 +653,121 @@ mod tests { assert!(reason.excludes()); } } + + fn counted(endpoint: &str, namespace: &str, producer: &str, value: f64) -> Observation { + Observation { + metric: "llm_d_epp_inflight_requests".to_owned(), + labels: BTreeMap::from([ + ("endpoint_name".to_owned(), endpoint.to_owned()), + ("namespace".to_owned(), namespace.to_owned()), + ("producer_name".to_owned(), producer.to_owned()), + ]), + value, + timestamp_ms: None, + } + } + + fn queued(pool: &str, value: f64) -> Observation { + Observation { + metric: "llm_d_epp_flow_control_queue_size".to_owned(), + labels: BTreeMap::from([("inference_pool".to_owned(), pool.to_owned())]), + value, + timestamp_ms: None, + } + } + + #[test] + fn in_flight_reads_the_larger_of_the_epp_count_and_the_pool_averages() { + let averages = [ + sample("llm_d_epp_average_running_requests", "qwen3", 6.0), + sample("llm_d_epp_average_queue_size", "qwen3", 1.5), + ]; + let mut observations = vec![counted("a", "ns", "p", 7.0), counted("b", "ns", "p", 5.0)]; + assert_eq!( + in_flight(&observations, Some(2.0), None), + Some((12.0, InFlightSource::Epp)), + "the EPP's count alone" + ); + observations.extend(averages.clone()); + assert_eq!( + in_flight(&observations, Some(1.0), Some("qwen3")), + Some((12.0, InFlightSource::Epp)), + "12 counted is above (6 + 1.5) x 1" + ); + let restarted = [vec![counted("a", "ns", "p", 0.0)], averages.to_vec()].concat(); + assert_eq!( + in_flight(&restarted, Some(2.0), Some("qwen3")), + Some((15.0, InFlightSource::EngineAverages)), + "an EPP restart reading 0 is floored by (6 + 1.5) x 2" + ); + assert_eq!( + in_flight(&averages, Some(2.0), Some("qwen3")), + Some((15.0, InFlightSource::EngineAverages)), + "the averages alone" + ); + } + + #[test] + fn in_flight_counts_each_endpoint_once_across_producers() { + let observations = [ + counted("a", "ns", "first", 4.0), + counted("a", "ns", "second", 3.0), + counted("b", "ns", "first", 2.0), + ]; + assert_eq!( + in_flight(&observations, None, None), + Some((6.0, InFlightSource::Epp)), + "4 + 2, not 9" + ); + } + + #[test] + fn in_flight_adds_requests_flow_control_holds_for_the_pool() { + let observations = [counted("a", "ns", "p", 3.0), queued("qwen3", 4.0), queued("other", 9.0)]; + assert_eq!( + in_flight(&observations, None, Some("qwen3")), + Some((7.0, InFlightSource::Epp)), + "3 dispatched plus 4 held for qwen3, not other's 9" + ); + } + + #[test] + fn in_flight_uses_pool_averages_when_the_epp_serves_several_pools() { + let observations = [ + counted("a", "ns", "p", 50.0), + sample("llm_d_epp_ready_endpoints", "qwen3", 2.0), + sample("llm_d_epp_ready_endpoints", "llama", 1.0), + sample("llm_d_epp_average_running_requests", "qwen3", 3.0), + sample("llm_d_epp_average_queue_size", "qwen3", 1.0), + ]; + assert_eq!( + in_flight(&observations, Some(2.0), Some("qwen3")), + Some((8.0, InFlightSource::EngineAverages)), + "the unlabeled count covers both pools, so (3 + 1) x 2 for qwen3" + ); + } + + #[test] + fn in_flight_is_unknown_rather_than_idle() { + let averages = [ + sample("llm_d_epp_average_running_requests", "qwen3", 6.0), + sample("llm_d_epp_average_queue_size", "qwen3", 1.5), + ]; + assert_eq!(in_flight(&averages, None, None), None, "no endpoint count, no estimate"); + assert_eq!( + in_flight(&averages, Some(0.0), Some("qwen3")), + None, + "no fresh endpoint behind frozen averages" + ); + assert_eq!( + in_flight(&averages[..1], Some(2.0), None), + None, + "a missing average is not zero" + ); + assert_eq!( + in_flight(&[queued("qwen3", 4.0)], None, Some("qwen3")), + None, + "a queue alone is no count" + ); + } } diff --git a/operator/src/resources/provider_metrics.rs b/operator/src/resources/provider_metrics.rs index cf9e96e9..b1c2f983 100644 --- a/operator/src/resources/provider_metrics.rs +++ b/operator/src/resources/provider_metrics.rs @@ -151,6 +151,7 @@ fn record_scrape( .as_ref() .and_then(|mc| mc.pool_name.as_deref()); let ready = crate::readiness::ready_endpoints(&parsed, plan.ready_names(), pool); + let in_flight = in_flight_observation(&parsed, ready, pool, plan.identity); let observations = parsed .into_iter() .filter(|o| plan.wanted.contains(o.metric.as_str())) @@ -160,10 +161,33 @@ fn record_scrape( o.timestamp_ms = None; o }) + .chain(in_flight) .collect(); readiness.record_success(key, ready, observations, Instant::now()); } +/// The resolved `grid_provider_in_flight_requests` sample, logging which source it came from. +fn in_flight_observation( + parsed: &[crate::signals::Observation], + ready: Option, + pool: Option<&str>, + identity: &str, +) -> Option { + let (value, source) = crate::readiness::in_flight(parsed, ready, pool)?; + tracing::debug!( + provider = identity, + in_flight = value, + ?source, + "signals: provider in-flight resolved" + ); + Some(crate::signals::Observation { + metric: crate::readiness::IN_FLIGHT_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value, + timestamp_ms: None, + }) +} + /// Scrape one provider's exposition, or `None` when TLS or the scrape fails. async fn scrape_provider_signals( provider: &InferenceProvider, diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index a90d3e8b..27e0e648 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -35,12 +35,16 @@ pub(crate) const MIN_WRITE_INTERVAL: Duration = Duration::from_secs(30); /// Store retention per series, seconds. const WINDOW_SECS: u64 = 60; -/// Freshness window the gateway orders over, milliseconds. -const LOAD_WINDOW_MS: i64 = 30_000; - /// Peer poll interval, the operator's default local scrape interval. const PEER_INTERVAL_MS: u64 = 5_000; +/// Freshness window the gateway orders over: two polls, milliseconds. +/// +/// The gateway orders by the worst reading in the window, so a site drained by a +/// spike returns two polls after it clears instead of thirty seconds. +#[expect(clippy::cast_possible_wrap, reason = "a few seconds fits i64")] +const LOAD_WINDOW_MS: i64 = 2 * PEER_INTERVAL_MS as i64; + /// Peer connect and request timeout, milliseconds. const PEER_TIMEOUT_MS: u64 = 2_000; From df00e7369cbd57f97e20a7998eab65ebc7bfb3da Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 15:06:02 -0400 Subject: [PATCH 04/10] feat(operator): publish a provider's recent latency and error ratio TTFT p50 and p90, TPOT, prefill seconds per token, and error ratio over the last 30s, from deltas of the EPP's request histograms, once 20 requests complete. Signed-off-by: Sam Batschelet --- docs/architecture/polling-metrics.md | 20 + operator/src/latency.rs | 458 +++++++++++++++++++++ operator/src/lib.rs | 3 +- operator/src/readiness.rs | 19 + operator/src/resources/provider_metrics.rs | 2 + 5 files changed, 501 insertions(+), 1 deletion(-) create mode 100644 operator/src/latency.rs diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index dc7421a9..ed0bf5f6 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -102,6 +102,26 @@ With no fresh endpoint it is unpublished, and the gateway reads it as unknown. With both known, the operator also publishes their ratio, requests held over capacity, as `grid_provider_saturation_ratio`. Gateways choose sites by it. +## Provider Latency + +Each site's operator also publishes recent latency per provider, from the EPP's request +histograms over the last 30s. A value is published only when at least 20 requests +completed in that window, and it is never borrowed from another site: + +- `grid_provider_ttft_p50_seconds` and `grid_provider_ttft_p90_seconds`: time to first + token for streaming requests, from the EPP's `llm_d_epp_request_ttft_seconds`. The EPP + times it from receiving the request, so it includes flow-control wait and network. +- `grid_provider_tpot_seconds`: mean time per output token for streaming requests. +- `grid_provider_prefill_seconds_per_token`: mean TTFT less mean flow-control wait, over + mean uncached input tokens. It is an estimate that moves with the workload mix: TTFT + counts streaming requests only, while token counts cover every request. +- `grid_provider_error_ratio`: failed requests over all requests. The latency histograms + record only successful requests, so a failing site can read fast. This shows it. + +The EPP labels these histograms by model, not pool, so an EPP serving more than one pool +reports their combined latency. An EPP restart resets its counters, and the window starts +over from the next scrape. + ## Failure Behavior | Condition | Signal produced | Routing effect | diff --git a/operator/src/latency.rs b/operator/src/latency.rs new file mode 100644 index 00000000..18c05a47 --- /dev/null +++ b/operator/src/latency.rs @@ -0,0 +1,458 @@ +//! A provider's recent request latency, from deltas of its EPP's cumulative histograms. +//! +//! A value with too few completed requests in the window is left unpublished, so it reads as unknown. + +use std::{ + collections::{BTreeMap, VecDeque}, + time::{Duration, Instant}, +}; + +use crate::signals::Observation; + +/// How far back the published latency looks. +pub(crate) const WINDOW: Duration = Duration::from_secs(30); + +/// Completed requests a value needs in the window before it is published. +pub(crate) const MIN_SAMPLES: f64 = 20.0; + +/// Time to first token at the median, streaming requests only, seconds. +pub const TTFT_P50_SIGNAL: &str = "grid_provider_ttft_p50_seconds"; +/// Time to first token at the 90th percentile, streaming requests only, seconds. +pub const TTFT_P90_SIGNAL: &str = "grid_provider_ttft_p90_seconds"; +/// Mean time per output token, streaming requests only, seconds. +pub const TPOT_SIGNAL: &str = "grid_provider_tpot_seconds"; +/// Estimated prefill seconds per uncached input token. +pub const PREFILL_SIGNAL: &str = "grid_provider_prefill_seconds_per_token"; +/// Failed requests over all requests. +pub const ERROR_RATIO_SIGNAL: &str = "grid_provider_error_ratio"; + +/// EPP time to first token, labeled `streaming`. +const TTFT: &str = "llm_d_epp_request_ttft_seconds"; +/// EPP time per output token, streaming requests. +const TPOT: &str = "llm_d_epp_request_streaming_tpot_seconds"; +/// EPP input tokens per request. +const INPUT_TOKENS: &str = "llm_d_epp_request_input_tokens"; +/// EPP cached prompt tokens per request, when the model server reports them. +const CACHED_TOKENS: &str = "llm_d_epp_request_cached_tokens"; +/// EPP flow-control wait, labeled `inference_pool`, present only with flow control on. +const QUEUE_WAIT: &str = "llm_d_epp_flow_control_request_queue_duration_seconds"; +/// EPP requests processed. +const REQUESTS: &str = "llm_d_epp_request_total"; +/// EPP requests that failed. +const ERRORS: &str = "llm_d_epp_request_error_total"; + +/// A histogram's `_sum` and `_count`. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +struct SumCount { + /// Summed observations. + sum: f64, + /// Observations. + count: f64, +} + +impl SumCount { + /// `self - earlier`, `None` when a counter went backwards, as after an EPP restart. + fn since(self, earlier: Self) -> Option { + (self.sum >= earlier.sum && self.count >= earlier.count).then_some(Self { + sum: self.sum - earlier.sum, + count: self.count - earlier.count, + }) + } + + /// The mean observation, `None` with none. + fn mean(self) -> Option { + (self.count > 0.0).then(|| self.sum / self.count) + } +} + +/// The EPP's counters summed over models, fairness and priority: cumulative at one scrape, +/// or the difference across a window. +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct Snapshot { + /// Streaming TTFT bucket counts by upper bound, in bound order. + ttft_buckets: Vec<(f64, f64)>, + /// Streaming TTFT. + ttft: SumCount, + /// Streaming time per output token. + tpot: SumCount, + /// Input tokens per request. + input: SumCount, + /// Cached tokens, absent until a response reports them. + cached: Option, + /// Flow-control wait for the pool, absent without flow control. + queue_wait: Option, + /// Requests processed. + requests: f64, + /// Requests failed. + errors: f64, +} + +/// The sum of `metric` over the observations `keep` accepts, `None` when none match. +fn summed(observations: &[Observation], metric: &str, keep: impl Fn(&Observation) -> bool) -> Option { + observations + .iter() + .filter(|o| o.metric == metric && keep(o)) + .map(|o| o.value) + .reduce(|a, b| a + b) +} + +/// The `_sum` and `_count` of histogram `name` over the observations `keep` accepts. +fn sum_count(observations: &[Observation], name: &str, keep: impl Fn(&Observation) -> bool + Copy) -> Option { + Some(SumCount { + sum: summed(observations, &format!("{name}_sum"), keep)?, + count: summed(observations, &format!("{name}_count"), keep)?, + }) +} + +/// Whether a TTFT series counts streaming requests. +fn streaming(o: &Observation) -> bool { + o.labels.get("streaming").is_some_and(|value| value == "true") +} + +/// Streaming TTFT bucket counts by upper bound, summed across series. +fn ttft_buckets(observations: &[Observation]) -> Vec<(f64, f64)> { + let mut buckets: BTreeMap = BTreeMap::new(); + let bucket = format!("{TTFT}_bucket"); + for o in observations.iter().filter(|o| o.metric == bucket && streaming(o)) { + if let Some(bound) = o.labels.get("le").and_then(|le| le.parse::().ok()) { + // Bounds are positive, so their bit patterns sort as the values do. + buckets.entry(bound.to_bits()).or_insert((bound, 0.0)).1 += o.value; + } + } + buckets.into_values().collect() +} + +impl Snapshot { + /// The cumulative counters in one scrape, flow-control wait read for `pool`. + pub(crate) fn of(observations: &[Observation], pool: Option<&str>) -> Self { + let any = |_: &Observation| true; + let in_pool = + |o: &Observation| pool.is_none_or(|pool| o.labels.get("inference_pool").is_none_or(|p| p == pool)); + Self { + ttft_buckets: ttft_buckets(observations), + ttft: sum_count(observations, TTFT, streaming).unwrap_or_default(), + tpot: sum_count(observations, TPOT, any).unwrap_or_default(), + input: sum_count(observations, INPUT_TOKENS, any).unwrap_or_default(), + cached: sum_count(observations, CACHED_TOKENS, any), + queue_wait: sum_count(observations, QUEUE_WAIT, in_pool), + requests: summed(observations, REQUESTS, any).unwrap_or(0.0), + errors: summed(observations, ERRORS, any).unwrap_or(0.0), + } + } + + /// `self - earlier`, `None` when any counter went backwards. + fn since(&self, earlier: &Self) -> Option { + let optional = |now: Option, then: Option| match (now, then) { + (Some(now), Some(then)) => now.since(then).map(Some), + (Some(now), None) => Some(Some(now)), + (None, _) => Some(None), + }; + (self.requests >= earlier.requests && self.errors >= earlier.errors).then_some(())?; + Some(Self { + ttft_buckets: self.bucket_deltas(earlier)?, + ttft: self.ttft.since(earlier.ttft)?, + tpot: self.tpot.since(earlier.tpot)?, + input: self.input.since(earlier.input)?, + cached: optional(self.cached, earlier.cached)?, + queue_wait: optional(self.queue_wait, earlier.queue_wait)?, + requests: self.requests - earlier.requests, + errors: self.errors - earlier.errors, + }) + } + + /// Each TTFT bucket's count less `earlier`'s, `None` when one went backwards. + fn bucket_deltas(&self, earlier: &Self) -> Option> { + self.ttft_buckets + .iter() + .map(|(bound, count)| { + let then = earlier + .ttft_buckets + .iter() + .find(|(b, _)| b.to_bits() == bound.to_bits()) + .map_or(0.0, |(_, earlier_count)| *earlier_count); + (*count >= then).then_some((*bound, count - then)) + }) + .collect() + } + + /// The series this window has enough samples to publish. + fn observations(&self) -> Vec { + let enough = |count: f64| count >= MIN_SAMPLES; + let mut published = Vec::new(); + if enough(self.ttft.count) { + published.extend(quantile(&self.ttft_buckets, 0.5).map(|value| sample(TTFT_P50_SIGNAL, value))); + published.extend(quantile(&self.ttft_buckets, 0.9).map(|value| sample(TTFT_P90_SIGNAL, value))); + } + if enough(self.tpot.count) { + published.extend(self.tpot.mean().map(|value| sample(TPOT_SIGNAL, value))); + } + if enough(self.ttft.count) && enough(self.input.count) { + published.extend( + self.prefill_seconds_per_token() + .map(|value| sample(PREFILL_SIGNAL, value)), + ); + } + if enough(self.requests) { + published.push(sample( + ERROR_RATIO_SIGNAL, + (self.errors / self.requests).clamp(0.0, 1.0), + )); + } + published + } + + /// Mean TTFT less mean flow-control wait, over mean uncached input tokens. + /// + /// Sensitive to the workload mix: TTFT counts streaming requests only, while token + /// counts cover every request. + fn prefill_seconds_per_token(&self) -> Option { + let wait = self.queue_wait.and_then(SumCount::mean).unwrap_or(0.0); + let cached = self.cached.map_or(0.0, |cached| cached.sum / self.input.count); + let uncached = self.input.mean()? - cached; + let prefill = self.ttft.mean()? - wait; + (uncached >= 1.0 && prefill > 0.0).then(|| prefill / uncached) + } +} + +/// One provider's recent cumulative snapshots. +#[derive(Clone, Debug, Default)] +pub(crate) struct History { + /// Snapshots by scrape time, oldest first, one at or before the window's start. + snapshots: VecDeque<(Instant, Snapshot)>, +} + +impl History { + /// Record this scrape and return the latency series to publish for it. + pub(crate) fn record(&mut self, snapshot: Snapshot, now: Instant) -> Vec { + while self + .snapshots + .get(1) + .is_some_and(|(at, _)| now.saturating_duration_since(*at) >= WINDOW) + { + self.snapshots.pop_front(); + } + let published = self + .snapshots + .front() + .and_then(|(_, oldest)| snapshot.since(oldest)) + .map_or_else(Vec::new, |window| window.observations()); + if self + .snapshots + .back() + .is_some_and(|(_, last)| snapshot.since(last).is_none()) + { + // A counter reset: start over from this scrape. + self.snapshots.clear(); + } + self.snapshots.push_back((now, snapshot)); + published + } +} + +/// The `q` quantile of cumulative bucket counts by upper bound, interpolated within its bucket. +fn quantile(buckets: &[(f64, f64)], q: f64) -> Option { + let total = buckets.last()?.1; + if total <= 0.0 { + return None; + } + let rank = q * total; + let mut lower = (0.0, 0.0); + for (bound, count) in buckets { + if *count >= rank { + if bound.is_infinite() { + return Some(lower.0); + } + let within = count - lower.1; + let fraction = if within > 0.0 { (rank - lower.1) / within } else { 1.0 }; + return Some(lower.0 + (bound - lower.0) * fraction); + } + lower = (*bound, *count); + } + Some(lower.0) +} + +/// An unlabeled sample of `metric`. +fn sample(metric: &str, value: f64) -> Observation { + Observation { + metric: metric.to_owned(), + labels: BTreeMap::new(), + value, + timestamp_ms: None, + } +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::float_arithmetic, + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + clippy::too_many_arguments, + reason = "tests" +)] +mod tests { + use super::*; + + impl History { + /// Record `observations` as one scrape, as the readiness store does. + fn record_observations( + &mut self, + observations: &[Observation], + pool: Option<&str>, + now: Instant, + ) -> Vec { + self.record(Snapshot::of(observations, pool), now) + } + } + + fn obs(metric: &str, labels: &[(&str, &str)], value: f64) -> Observation { + Observation { + metric: metric.to_owned(), + labels: labels.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned())).collect(), + value, + timestamp_ms: None, + } + } + + /// An EPP exposition after `requests` streaming requests, each with TTFT `ttft`, TPOT + /// `tpot`, `input` tokens of which `cached` were cached, and `errors` failures. + fn scrape(requests: f64, ttft: f64, tpot: f64, input: f64, cached: f64, errors: f64) -> Vec { + let streaming = [("streaming", "true"), ("model_name", "m")]; + let mut out = vec![]; + for bound in [0.1, 0.2, 0.4, 0.8, 1.6, f64::INFINITY] { + let le = if bound.is_infinite() { + "+Inf".to_owned() + } else { + bound.to_string() + }; + let count = if ttft <= bound { requests } else { 0.0 }; + out.push(obs( + &format!("{TTFT}_bucket"), + &[streaming[0], streaming[1], ("le", &le)], + count, + )); + } + out.push(obs(&format!("{TTFT}_sum"), &streaming, ttft * requests)); + out.push(obs(&format!("{TTFT}_count"), &streaming, requests)); + // A non-streaming series must not count. + out.push(obs(&format!("{TTFT}_sum"), &[("streaming", "false")], 999.0)); + out.push(obs(&format!("{TTFT}_count"), &[("streaming", "false")], 1.0)); + out.push(obs(&format!("{TPOT}_sum"), &[], tpot * requests)); + out.push(obs(&format!("{TPOT}_count"), &[], requests)); + out.push(obs(&format!("{INPUT_TOKENS}_sum"), &[], input * requests)); + out.push(obs(&format!("{INPUT_TOKENS}_count"), &[], requests)); + out.push(obs(&format!("{CACHED_TOKENS}_sum"), &[], cached * requests)); + out.push(obs(&format!("{CACHED_TOKENS}_count"), &[], requests)); + out.push(obs(REQUESTS, &[], requests)); + out.push(obs(ERRORS, &[("error_code", "500")], errors)); + out + } + + fn value(published: &[Observation], metric: &str) -> Option { + published.iter().find(|o| o.metric == metric).map(|o| o.value) + } + + #[test] + fn a_window_publishes_latency_from_the_requests_inside_it() { + let mut history = History::default(); + let start = Instant::now(); + let first = history.record_observations(&scrape(100.0, 0.3, 0.02, 1000.0, 200.0, 0.0), None, start); + assert!(first.is_empty(), "a first scrape has no window"); + let later = scrape(200.0, 0.3, 0.02, 1000.0, 200.0, 5.0); + let published = history.record_observations(&later, None, start + Duration::from_secs(10)); + let p50 = value(&published, TTFT_P50_SIGNAL).unwrap(); + assert!( + (0.2..=0.4).contains(&p50), + "every request in the 0.2..0.4 bucket: {p50}" + ); + assert!(value(&published, TTFT_P90_SIGNAL).unwrap() >= p50); + assert!((value(&published, TPOT_SIGNAL).unwrap() - 0.02).abs() < 1e-9); + let prefill = value(&published, PREFILL_SIGNAL).unwrap(); + assert!( + (prefill - 0.3 / 800.0).abs() < 1e-9, + "0.3s over 800 uncached tokens: {prefill}" + ); + assert!((value(&published, ERROR_RATIO_SIGNAL).unwrap() - 0.05).abs() < 1e-9); + } + + #[test] + fn too_few_requests_publish_nothing() { + let mut history = History::default(); + let start = Instant::now(); + history.record_observations(&scrape(100.0, 0.3, 0.02, 1000.0, 0.0, 0.0), None, start); + let published = history.record_observations( + &scrape(105.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(10), + ); + assert!(published.is_empty(), "5 requests is too few: {published:?}"); + } + + #[test] + fn a_counter_reset_publishes_nothing_and_starts_over() { + let mut history = History::default(); + let start = Instant::now(); + history.record_observations(&scrape(500.0, 0.3, 0.02, 1000.0, 0.0, 0.0), None, start); + let reset = history.record_observations( + &scrape(50.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(5), + ); + assert!(reset.is_empty(), "an EPP restart is not negative latency"); + let after = history.record_observations( + &scrape(100.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(10), + ); + assert!(value(&after, TPOT_SIGNAL).is_some(), "the window restarts at the reset"); + } + + #[test] + fn the_window_drops_snapshots_older_than_it() { + let mut history = History::default(); + let start = Instant::now(); + for (seconds, requests) in [(0, 0.0), (20, 100.0), (40, 100.0)] { + history.record_observations( + &scrape(requests, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(seconds), + ); + } + let idle = history.record_observations( + &scrape(100.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(60), + ); + assert!(idle.is_empty(), "nothing completed in the last 30s: {idle:?}"); + } + + #[test] + fn queue_wait_comes_off_ttft_for_prefill() { + let window = Snapshot { + ttft: SumCount { + sum: 50.0, + count: 100.0, + }, + input: SumCount { + sum: 100_000.0, + count: 100.0, + }, + queue_wait: Some(SumCount { + sum: 20.0, + count: 100.0, + }), + requests: 100.0, + ..Snapshot::default() + }; + let prefill = window.prefill_seconds_per_token().unwrap(); + assert!((prefill - 0.3 / 1000.0).abs() < 1e-12, "(0.5 - 0.2) / 1000: {prefill}"); + } + + #[test] + fn quantiles_interpolate_within_a_bucket() { + let buckets = [(1.0, 0.0), (2.0, 10.0), (f64::INFINITY, 10.0)]; + assert!((quantile(&buckets, 0.5).unwrap() - 1.5).abs() < 1e-9); + assert_eq!(quantile(&[(1.0, 0.0), (f64::INFINITY, 0.0)], 0.5), None); + } +} diff --git a/operator/src/lib.rs b/operator/src/lib.rs index 4cb3fdc3..bfc6b0be 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -31,6 +31,8 @@ pub mod crd; pub mod enroll; /// Operator error types. pub mod error; +/// Provider readiness resolved from scraped metrics. +pub mod latency; /// Prometheus metrics for gateway probe and phase-transition observability. pub mod metrics; /// Pure Prometheus text-format parser for inference backend metrics. @@ -41,7 +43,6 @@ pub mod metrics_scraper; pub mod metrics_tls; /// Short-lived tokens for the metrics scraper ServiceAccount. pub(crate) mod metrics_token; -/// Provider readiness resolved from scraped metrics. pub mod readiness; /// Kubernetes resource builders. pub mod resources; diff --git a/operator/src/readiness.rs b/operator/src/readiness.rs index cee161a6..debb23b4 100644 --- a/operator/src/readiness.rs +++ b/operator/src/readiness.rs @@ -135,6 +135,8 @@ struct Probe { ready_streak: u32, /// Whether the count marks the provider not ready, with [`STREAK`] hysteresis both ways. no_endpoints: bool, + /// Recent EPP latency snapshots. + latency: crate::latency::History, } /// One provider's verdict and the detail behind it. @@ -161,6 +163,23 @@ impl ReadinessStore { self.0.lock().unwrap_or_else(PoisonError::into_inner) } + /// Record a scrape's latency counters and return the latency series it publishes. + pub(crate) fn record_latency( + &self, + key: &str, + observations: &[Observation], + pool: Option<&str>, + now: Instant, + ) -> Vec { + // Built before locking, so the store is held only to append it. + let snapshot = crate::latency::Snapshot::of(observations, pool); + self.probes() + .entry(key.to_owned()) + .or_default() + .latency + .record(snapshot, now) + } + /// Record a successful scrape at `now`. #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] pub(crate) fn record_success( diff --git a/operator/src/resources/provider_metrics.rs b/operator/src/resources/provider_metrics.rs index b1c2f983..3fed9a60 100644 --- a/operator/src/resources/provider_metrics.rs +++ b/operator/src/resources/provider_metrics.rs @@ -152,6 +152,7 @@ fn record_scrape( .and_then(|mc| mc.pool_name.as_deref()); let ready = crate::readiness::ready_endpoints(&parsed, plan.ready_names(), pool); let in_flight = in_flight_observation(&parsed, ready, pool, plan.identity); + let latency = readiness.record_latency(key, &parsed, pool, Instant::now()); let observations = parsed .into_iter() .filter(|o| plan.wanted.contains(o.metric.as_str())) @@ -162,6 +163,7 @@ fn record_scrape( o }) .chain(in_flight) + .chain(latency) .collect(); readiness.record_success(key, ready, observations, Instant::now()); } From 95cf6d7a7ce64c61806181c49fa1a5de0af2d80f Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 15:13:59 -0400 Subject: [PATCH 05/10] feat(operator): export provider series on /metrics and bound what a hub keeps from peers The provider series are gauges on /metrics, labeled grid_site and grid_provider, this site's and its peers', each in its own family. A hub keeps from a peer only the contract names and the EPP pool averages, DNS-1123 grid_provider values, finite non-negative values, and 64 providers; refusals count in grid_peer_signals_refused_total{peer,reason}. Signed-off-by: Sam Batschelet --- docs/architecture/polling-metrics.md | 10 ++ operator/src/main.rs | 2 + operator/src/metrics.rs | 239 +++++++++++++++++++++++++++ operator/src/signals.rs | 180 +++++++++++++++++++- 4 files changed, 429 insertions(+), 2 deletions(-) diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index ed0bf5f6..5843714d 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -21,6 +21,7 @@ one verified identity. | Verify | The peer's certificate is checked against the Grid CA and its SPIFFE identity, then the verified identity is compared to the site the poller intended to reach. A valid Grid peer answering for a site the poller did not dial is refused. | | Read | The exposition body is read under a byte ceiling and a time bound, so a slow or oversized peer cannot hold the poll open or exhaust memory. | | Store | Each reading is keyed on the verified peer identity, never a value the response body carries. A body label that disagrees with the verified owner is dropped. | +| Bound | An operator keeps from a peer only the signal contract names and the EPP pool averages the gateway routes on, a `grid_provider` that is a DNS-1123 label, a finite non-negative value (at most one for `grid_provider_ready` and `grid_provider_error_ratio`), and its first 64 providers in name order. A peer's custom `signalNames` are dropped. Each refusal counts in `grid_peer_signals_refused_total{peer,reason}`. | The verified peer identity is the store key. The response body cannot choose where its readings land, so one peer cannot inject readings attributed to @@ -122,6 +123,15 @@ The EPP labels these histograms by model, not pool, so an EPP serving more than reports their combined latency. An EPP restart resets its counters, and the window starts over from the next scrape. +## Provider Series on /metrics + +The operator also exports the provider series on its Prometheus `/metrics` listener, +labeled `grid_site` and `grid_provider`: readiness, in-flight requests, capacity, +saturation, and the latency series above. It exports its own providers and those it +polls from peers, so a Prometheus that scrapes one hub sees every site. A peer's value +there is what the hub last polled, up to one peer poll old, not what the peer's own +operator holds now. A series the operator does not hold is absent, not 0. + ## Failure Behavior | Condition | Signal produced | Routing effect | diff --git a/operator/src/main.rs b/operator/src/main.rs index d5a2a5e3..36ca891a 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -181,6 +181,8 @@ async fn main() { .with_scrape_interval(config.signals.scrape_interval()) .hold_membership(), ); + // This site's providers first, then what its peers publish, on /metrics. + operator::metrics::register_provider_signals(vec![ctx.signals(), ctx.peers()]); // Controllers run now. Only SWIM and what dials peers wait on the advertise address. let (swim_tx, swim_rx) = tokio::sync::watch::channel(SwimStage::Starting); diff --git a/operator/src/metrics.rs b/operator/src/metrics.rs index 999cbad8..fed4efb4 100644 --- a/operator/src/metrics.rs +++ b/operator/src/metrics.rs @@ -39,6 +39,8 @@ static REGISTRY: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_RETRIES.clone())) .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(PEER_SIGNALS_REFUSED.clone())) + .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_DURATION.clone())) .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_SLOW.clone())) @@ -125,6 +127,15 @@ static PEER_POLL_TOTAL: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()) }); +/// Peer observations refused at ingest, by peer and reason. +static PEER_SIGNALS_REFUSED: LazyLock = LazyLock::new(|| { + IntCounterVec::new( + Opts::new("grid_peer_signals_refused_total", "Peer observations refused at ingest"), + &["peer", "reason"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + /// Retried attempts by peer and the outcome that prompted the retry. /// /// Separate from the poll counter because a poll that succeeded on its third @@ -427,6 +438,11 @@ pub(crate) fn record_peer_poll(peer: &str, outcome: &str, duration: Duration, by } } +/// Count a peer observation this hub refused: `name`, `provider`, `value`, or `provider_cap`. +pub(crate) fn record_peer_signal_refused(peer: &str, reason: &str) { + PEER_SIGNALS_REFUSED.with_label_values(&[peer, reason]).inc(); +} + /// Record an attempt that failed and will be tried again. pub(crate) fn record_peer_retry(peer: &str, reason: &str) { PEER_POLL_RETRIES.with_label_values(&[peer, reason]).inc(); @@ -476,6 +492,116 @@ pub fn record_site_identity_renewal(result: &str) { SITE_IDENTITY_RENEWALS.with_label_values(&[result]).inc(); } +/// Provider series exported on `/metrics`, as each site's operator resolves them. +const PROVIDER_SIGNALS: [(&str, &str); 9] = [ + ("grid_provider_ready", "1 when the provider can serve, 0 when not."), + ( + "grid_provider_in_flight_requests", + "Requests the provider holds, running, engine-queued, and held by flow control.", + ), + ( + "grid_provider_capacity_requests", + "maxRunning per endpoint times fresh ready endpoints.", + ), + ("grid_provider_saturation_ratio", "Requests held over capacity."), + ( + "grid_provider_ttft_p50_seconds", + "Median streaming time to first token over the last 30s.", + ), + ( + "grid_provider_ttft_p90_seconds", + "90th percentile streaming time to first token over the last 30s.", + ), + ( + "grid_provider_tpot_seconds", + "Mean streaming time per output token over the last 30s.", + ), + ( + "grid_provider_prefill_seconds_per_token", + "Estimated prefill seconds per uncached input token over the last 30s. Moves with the workload mix.", + ), + ( + "grid_provider_error_ratio", + "Failed requests over all requests in the last 30s.", + ), +]; + +/// Labels on each exported provider series: both bounded by the grid's sites and providers. +const PROVIDER_LABELS: [&str; 2] = ["grid_site", "grid_provider"]; + +/// Exports the provider series held in `stores` at each scrape, this site's own and those +/// polled from peers. A series this site does not hold is absent, not 0. +struct ProviderSignals { + /// The local and peer signal stores. + stores: Vec, + /// One gauge per exported series, paired with its metric name so a failed constructor + /// drops only its own series. + gauges: Vec<(&'static str, prometheus::GaugeVec)>, + /// Serializes a scrape's reset and refill of the shared gauges. + collecting: std::sync::Mutex<()>, +} + +impl prometheus::core::Collector for ProviderSignals { + fn desc(&self) -> Vec<&prometheus::core::Desc> { + self.gauges.iter().flat_map(|(_, gauge)| gauge.desc()).collect() + } + + fn collect(&self) -> Vec { + let _collecting = self + .collecting + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for (_, gauge) in &self.gauges { + gauge.reset(); + } + let mut seen = std::collections::BTreeSet::new(); + for sample in self.stores.iter().flat_map(crate::signals::SignalStore::current) { + let (Some(site), Some(provider)) = (sample.labels.get("grid_site"), sample.labels.get("grid_provider")) + else { + continue; + }; + let Some((_, gauge)) = self.gauges.iter().find(|(name, _)| *name == sample.metric) else { + continue; + }; + // The first store holding a series wins: this site's own before a peer's. + if seen.insert((sample.metric.clone(), site.clone(), provider.clone())) { + gauge + .with_label_values(&[site.as_str(), provider.as_str()]) + .set(sample.value); + } + } + self.gauges + .iter() + .flat_map(|(_, gauge)| prometheus::core::Collector::collect(gauge)) + .collect() + } +} + +/// Export the provider series held in `stores` on `/metrics`. Call once, with this site's +/// store first. +pub fn register_provider_signals(stores: Vec) { + if let Err(error) = REGISTRY.register(Box::new(provider_signals(stores))) { + tracing::warn!(%error, "metrics: provider signals already registered"); + } +} + +/// The collector over `stores`. +fn provider_signals(stores: Vec) -> ProviderSignals { + let gauges = PROVIDER_SIGNALS + .iter() + .filter_map(|(name, help)| { + prometheus::GaugeVec::new(Opts::new(*name, *help), &PROVIDER_LABELS) + .ok() + .map(|gauge| (*name, gauge)) + }) + .collect(); + ProviderSignals { + stores, + gauges, + collecting: std::sync::Mutex::new(()), + } +} + /// Gather all registered metrics for serialization. pub(crate) fn gather_metrics() -> Vec { expire_site_phases(); @@ -645,4 +771,117 @@ mod tests { "output should contain duration histogram" ); } + + #[test] + #[expect(clippy::too_many_lines, reason = "one store across two scrapes")] + fn provider_signals_drop_a_series_the_store_no_longer_holds() { + use prometheus::core::Collector as _; + let ready = crate::signals::Observation { + metric: "grid_provider_ready".to_owned(), + labels: [("grid_site", "hq"), ("grid_provider", "pool")] + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(), + value: 1.0, + timestamp_ms: None, + }; + let store = crate::signals::SignalStore::new(); + let collector = provider_signals(vec![store.clone()]); + let count = |families: &[MetricFamily]| -> usize { + families + .iter() + .filter(|family| family.name() == "grid_provider_ready") + .map(|family| family.get_metric().len()) + .sum() + }; + store.refresh( + std::collections::BTreeMap::from([("pool".to_owned(), vec![ready])]), + Duration::from_secs(60), + ); + assert_eq!(count(&collector.collect()), 1, "a held series is exported"); + store.refresh( + std::collections::BTreeMap::from([("pool".to_owned(), Vec::new())]), + Duration::from_secs(60), + ); + assert_eq!( + count(&collector.collect()), + 0, + "a dropped series is not exported from the last scrape" + ); + assert_eq!( + collector.desc().len(), + PROVIDER_SIGNALS.len(), + "one desc per exported series" + ); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one scenario across both stores")] + fn provider_signals_export_this_site_and_its_peers_and_omit_what_is_not_held() { + use prometheus::core::Collector as _; + let held = |site: &str, metric: &str, value: f64| crate::signals::Observation { + metric: metric.to_owned(), + labels: [("grid_site", site), ("grid_provider", "pool")] + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(), + value, + timestamp_ms: None, + }; + let local = crate::signals::SignalStore::new(); + local.refresh( + std::collections::BTreeMap::from([( + "pool".to_owned(), + vec![ + held("hq", "grid_provider_saturation_ratio", 0.5), + held("hq", "grid_provider_ttft_p50_seconds", 0.2), + held("hq", "llm_d_epp_average_queue_size", 3.0), + ], + )]), + Duration::from_secs(60), + ); + let peers = crate::signals::SignalStore::new(); + peers.refresh( + std::collections::BTreeMap::from([( + "retail".to_owned(), + vec![ + held("retail", "grid_provider_saturation_ratio", 0.9), + held("hq", "grid_provider_saturation_ratio", 7.0), + ], + )]), + Duration::from_secs(60), + ); + let families = provider_signals(vec![local, peers]).collect(); + let series = |name: &str| -> Vec<(String, f64)> { + families + .iter() + .filter(|family| family.name() == name) + .flat_map(|family| family.get_metric().iter()) + .map(|metric| { + let site = metric + .get_label() + .iter() + .find(|label| label.name() == "grid_site") + .map(|label| label.value().to_owned()) + .unwrap_or_default(); + (site, metric.get_gauge().value()) + }) + .collect() + }; + let mut saturation = series("grid_provider_saturation_ratio"); + saturation.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + saturation, + [("hq".to_owned(), 0.5), ("retail".to_owned(), 0.9)], + "this site's value wins" + ); + assert_eq!(series("grid_provider_ttft_p50_seconds"), [("hq".to_owned(), 0.2)]); + assert!(series("grid_provider_tpot_seconds").is_empty(), "not held, not 0"); + assert!( + families + .iter() + .all(|family| family.name() != "llm_d_epp_average_queue_size"), + "only provider series" + ); + } } diff --git a/operator/src/signals.rs b/operator/src/signals.rs index 63ffc569..39427f6c 100644 --- a/operator/src/signals.rs +++ b/operator/src/signals.rs @@ -479,6 +479,20 @@ impl SignalStore { (out, oldest) } + /// Every sample still served, across targets. + #[must_use] + pub fn current(&self) -> Vec { + let Ok(guard) = self.inner.read() else { + return Vec::new(); + }; + let now = Instant::now(); + guard + .values() + .filter(|held| held.expires_at > now) + .flat_map(|held| held.samples.iter().cloned()) + .collect() + } + /// Targets currently held and unexpired. #[must_use] pub fn targets(&self) -> Vec { @@ -1105,7 +1119,7 @@ impl PollPeers { .map(|(peer, url, pins)| async move { let (body, date) = self.poll_one(&peer, &url, &pins).await?; let now_ms = wall_millis(SystemTime::now(), Duration::ZERO); - let mut observations = retain_origin(parse(&body), &peer); + let mut observations = bound_peer(retain_origin(parse(&body), &peer), &peer); reexpress_peer_ages(&mut observations, date, now_ms); Some((peer, observations)) }); @@ -1207,6 +1221,86 @@ fn with_query(url: &str, query: &str) -> Option { http::Uri::from_parts(parts).ok().map(|uri| uri.to_string()) } +/// Names a hub keeps from a peer: the cross-site contract, plus the EPP pool averages and +/// ready counts the gateway routes on until it routes on saturation. A peer's custom +/// `signalNames` are dropped here. +const PEER_SIGNAL_NAMES: [&str; 16] = [ + crate::readiness::READY_SIGNAL, + crate::readiness::IN_FLIGHT_SIGNAL, + crate::readiness::CAPACITY_SIGNAL, + crate::readiness::SATURATION_SIGNAL, + crate::latency::TTFT_P50_SIGNAL, + crate::latency::TTFT_P90_SIGNAL, + crate::latency::TPOT_SIGNAL, + crate::latency::PREFILL_SIGNAL, + crate::latency::ERROR_RATIO_SIGNAL, + "inference_pool_average_queue_size", + "llm_d_epp_average_queue_size", + "inference_pool_average_running_requests", + "llm_d_epp_average_running_requests", + "inference_pool_average_kv_cache_utilization", + "llm_d_epp_average_kv_cache_utilization", + "llm_d_epp_ready_endpoints", +]; + +/// Most providers a hub keeps from one peer, so a peer cannot grow the hub's series without +/// bound. The first in name order are kept. +pub const MAX_PEER_PROVIDERS: usize = 64; + +/// Signals that are shares, which a peer may not report above one. +const UNIT_SIGNALS: [&str; 2] = [crate::readiness::READY_SIGNAL, crate::latency::ERROR_RATIO_SIGNAL]; + +/// A value a hub accepts: finite and non-negative, and at most one for a share. +fn plausible(observation: &Observation) -> bool { + observation.value.is_finite() + && observation.value >= 0.0 + && (observation.value <= 1.0 || !UNIT_SIGNALS.contains(&observation.metric.as_str())) +} + +/// Why a hub refuses `observation` from a peer, `None` when it accepts it. +fn refusal(observation: &Observation) -> Option<&'static str> { + if !PEER_SIGNAL_NAMES.contains(&observation.metric.as_str()) { + Some("name") + } else if observation + .labels + .get(PROVIDER_LABEL) + .is_none_or(|provider| certs::validate_site_name(provider).is_err()) + { + Some("provider") + } else if !plausible(observation) { + Some("value") + } else { + None + } +} + +/// Keep what a hub accepts from `peer`: an allowed name, a `grid_provider` that is a DNS-1123 +/// label, a plausible value, and at most [`MAX_PEER_PROVIDERS`] providers. Each refusal is +/// counted by reason. +fn bound_peer(observations: Vec, peer: &str) -> Vec { + let mut kept = Vec::with_capacity(observations.len()); + for observation in observations { + match refusal(&observation) { + Some(reason) => crate::metrics::record_peer_signal_refused(peer, reason), + None => kept.push(observation), + } + } + let providers: std::collections::BTreeSet<&str> = kept + .iter() + .filter_map(|o| o.labels.get(PROVIDER_LABEL).map(String::as_str)) + .collect(); + let Some(&last) = providers.iter().nth(MAX_PEER_PROVIDERS.saturating_sub(1)) else { + return kept; + }; + let last = last.to_owned(); + let before = kept.len(); + kept.retain(|o| o.labels.get(PROVIDER_LABEL).is_some_and(|p| *p <= last)); + for _ in kept.len()..before { + crate::metrics::record_peer_signal_refused(peer, "provider_cap"); + } + kept +} + /// Keep only the observations a peer made itself. /// /// A relayed copy is dropped rather than trusted, so every site's data reaches @@ -1258,6 +1352,88 @@ fn reexpress_peer_ages(observations: &mut [Observation], date: Option Observation { + Observation { + metric: metric.to_owned(), + labels: BTreeMap::from([ + (SITE_LABEL.to_owned(), "retail".to_owned()), + (PROVIDER_LABEL.to_owned(), provider.to_owned()), + ]), + value, + timestamp_ms: None, + } + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one table of accepted and refused samples")] + fn a_hub_refuses_peer_names_providers_and_values_outside_the_contract() { + let cases = [ + ( + "contract name", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", 0.5), + true, + ), + ( + "gateway load name", + peer_sample("inference_pool_average_queue_size", "pool", 3.0), + true, + ), + ("custom name", peer_sample("my_custom_queue", "pool", 3.0), false), + ( + "provider not a label", + peer_sample(crate::readiness::READY_SIGNAL, "Pool_1", 1.0), + false, + ), + ( + "not finite", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", f64::NAN), + false, + ), + ( + "infinite", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", f64::INFINITY), + false, + ), + ( + "negative", + peer_sample(crate::readiness::IN_FLIGHT_SIGNAL, "pool", -1.0), + false, + ), + ( + "share above one", + peer_sample(crate::latency::ERROR_RATIO_SIGNAL, "pool", 1.5), + false, + ), + ( + "saturation above one", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", 1.5), + true, + ), + ]; + for (label, observation, kept) in cases { + assert_eq!( + bound_peer(vec![observation], "retail").len(), + usize::from(kept), + "{label}" + ); + } + } + + #[test] + fn a_hub_keeps_at_most_the_first_providers_of_a_peer_in_name_order() { + let observations: Vec = (0..MAX_PEER_PROVIDERS + 5) + .map(|i| peer_sample(crate::readiness::READY_SIGNAL, &format!("p{i:03}"), 1.0)) + .collect(); + let kept = bound_peer(observations, "retail"); + assert_eq!(kept.len(), MAX_PEER_PROVIDERS); + assert!( + kept.iter() + .all(|o| o.labels.get(PROVIDER_LABEL).is_some_and(|p| p.as_str() < "p064")), + "the first {MAX_PEER_PROVIDERS} in name order are kept" + ); + } + #[test] fn peers_are_dialed_at_a_dialable_signals_endpoint() { let member = |site: &str, endpoint: &str, signals: Option<&str>| MemberRecord { @@ -1302,7 +1478,7 @@ mod tests { let mut request = [0_u8; 1024]; drop(stream.read(&mut request).await); drop(release.await); - let body = format!("load{{grid_site=\"{site}\"}} 1\n"); + let body = format!("inference_pool_average_queue_size{{grid_site=\"{site}\",grid_provider=\"pool\"}} 1\n"); let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()); drop(stream.write_all(response.as_bytes()).await); }); From f2ed6781bb3e60029b1be35c90baf64854ef1d53 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 17:49:19 -0400 Subject: [PATCH 06/10] feat(operator)!: say why a provider is not ready, and count every scrape Ready uses the status contract's reasons: NoEndpointsReady, NoLivenessCheck for a scrape without the pool's ready-endpoint series, ScrapeTimedOut, ScrapeUnauthorized, TLSHandshakeFailed or ScrapeFailed with the class in the message, and AwaitingFirstScrape (Unknown). A change logs WARN when the provider turns not ready. Scrapes count in grid_provider_scrape_total and grid_provider_last_scrape_success_timestamp_seconds. A not-ready provider keeps a ready=0 row. The mock provider exposes a ready count. Signed-off-by: Sam Batschelet --- docs/architecture/crds.md | 23 +- docs/architecture/polling-metrics.md | 8 +- mock-providers/src/openai.rs | 11 +- operator/src/controller/grid_network.rs | 52 +++- operator/src/metrics.rs | 77 ++++++ operator/src/metrics_scraper.rs | 124 ++++++++- operator/src/readiness.rs | 308 ++++++++++++++++----- operator/src/resources/provider_metrics.rs | 43 ++- operator/src/signals.rs | 1 + 9 files changed, 544 insertions(+), 103 deletions(-) diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 548a492b..81e41f98 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -565,17 +565,30 @@ credential projection can become available. **Readiness**: `status.conditions` carries a `Ready` condition, written by the operator's signals loop from each scrape of the provider's metrics. Readiness is -a condition, not a phase, so `phase` is unchanged by it. +a condition, not a phase: `phase` follows the provider's configuration only, and +says nothing about whether it serves now. | Status | Reason | When | |---|---|---| -| `True` | `Ready` | The latest scrape succeeded with at least one ready endpoint, or exposed no count. | -| `False` | `NoReadyEndpoints` | Two consecutive scrapes counted zero ready endpoints. | -| `False` | `MetricsUnreachable` | No scrape succeeded within `staleMetricsSeconds`, and the latest failed. | -| `False` | `MetricsStale` | No scrape succeeded within `staleMetricsSeconds`. | +| `True` | `Ready` | The latest scrape succeeded with at least one ready endpoint. | +| `False` | `NoEndpointsReady` | Two consecutive scrapes counted zero ready endpoints. | +| `False` | `NoLivenessCheck` | The scrape answered without the pool's ready-endpoint series (`llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`, for `poolName`). The message names what was missing. | +| `False` | `ScrapeTimedOut` | No scrape succeeded within `staleMetricsSeconds`, and the latest timed out. | +| `False` | `ScrapeUnauthorized` | As above, and the latest was refused with 401 or 403. | +| `False` | `TLSHandshakeFailed` | As above, and the latest failed TLS, including the TLS material. | +| `False` | `ScrapeFailed` | As above, and the latest failed otherwise. The message names the class: `dns`, `connect`, `http`, `body_cap`, `parse`, or `config`. | +| `False` | `MetricsStale` | No scrape succeeded within `staleMetricsSeconds`, and none failed. | | `False` | `ProviderUnavailable` | The provider is `Unavailable`. | +| `Unknown` | `AwaitingFirstScrape` | No scrape has succeeded yet, within the grace window. | | `Unknown` | `MetricsNotConfigured` | No `metricsConfig`, so readiness cannot be read. | +The operator logs each change of reason once: at WARN when the provider turns not +ready, at INFO when it returns to `Ready` or waits. Each scrape counts in +`grid_provider_scrape_total{grid_provider,result}`, where `result` is `success`, +`no_series`, or a failure class, and +`grid_provider_last_scrape_success_timestamp_seconds{grid_provider}` holds the time +of the last scrape with the ready-endpoint series. + A provider whose `Ready` is `False` is excluded from routing: its site publishes `grid_provider_ready 0`, and its serving config entry carries `admission: none`. See [Polling Cross-Site Load Signals](polling-metrics.md#provider-readiness). diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index 5843714d..385a7550 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -56,9 +56,11 @@ request path does not read raw signals or compute load. It reads resolved order. Each site's operator decides whether each of its providers can serve now and publishes the verdict as `grid_provider_ready{grid_site,grid_provider}`: 1 when ready, 0 when not. A provider is not ready when its EPP reports zero ready -endpoints for two scrapes in a row, when no scrape has succeeded within -`staleMetricsSeconds` (half the signal TTL when unset), or when the provider is -`Unavailable`. The same verdict is the provider's `Ready` condition. +endpoints for two scrapes in a row, when a scrape answers without the pool's +ready-endpoint series, when no scrape has succeeded within `staleMetricsSeconds` +(half the signal TTL when unset), or when the provider is `Unavailable`. The same +verdict is the provider's `Ready` condition, whose reason names the cause and, for a +failed scrape, its class. See crds.md for the reasons and the scrape metrics. The gateway reads the latest readiness sample for each candidate when it orders the snapshot. A candidate whose latest sample is 0 is excluded, so a site is diff --git a/mock-providers/src/openai.rs b/mock-providers/src/openai.rs index 04886a6f..57bdb354 100644 --- a/mock-providers/src/openai.rs +++ b/mock-providers/src/openai.rs @@ -159,7 +159,11 @@ async fn metrics(axum::extract::State(state): axum::extract::State) -> Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "text/plain; version=0.0.4") - .body(Body::from(format!("grid_demo_queue_depth {}\n", state.queue_depth))) + // A ready count, as an EPP exposes, so a grid operator reads the mock as ready. + .body(Body::from(format!( + "grid_demo_queue_depth {}\ninference_pool_ready_pods 1\n", + state.queue_depth + ))) .unwrap_or_default() } @@ -448,7 +452,10 @@ mod tests { let resp = send(req).await; assert_eq!(resp.status(), StatusCode::OK); let body = to_bytes(resp.into_body(), 4096).await.unwrap_or_default(); - assert_eq!(body.as_ref(), b"grid_demo_queue_depth 0.25\n"); + assert_eq!( + body.as_ref(), + b"grid_demo_queue_depth 0.25\ninference_pool_ready_pods 1\n" + ); } #[tokio::test] diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 09b1647a..bec88be5 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -635,13 +635,24 @@ async fn apply_ready_condition( &Patch::Apply(patch), )) .await?; - tracing::info!( - provider = name, - status = verdict.reason.status(), - reason = verdict.reason.as_str(), - message = %verdict.message, - "provider readiness changed" - ); + // A turn away from Ready warns; a return to Ready or a wait is informational. + if verdict.reason.excludes() { + tracing::warn!( + provider = name, + status = verdict.reason.status(), + reason = verdict.reason.as_str(), + message = %verdict.message, + "provider readiness changed" + ); + } else { + tracing::info!( + provider = name, + status = verdict.reason.status(), + reason = verdict.reason.as_str(), + message = %verdict.message, + "provider readiness changed" + ); + } Ok(()) } @@ -4066,6 +4077,27 @@ mod tests { } } + #[test] + fn a_provider_not_ready_publishes_ready_zero_without_a_fresh_scrape() { + let ready = |value: f64| signals::Observation { + metric: readiness::READY_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value, + timestamp_ms: None, + }; + assert!( + matches!( + published_signals(None, ready(0.0), None, true).as_deref(), + Some([only]) if only.metric == readiness::READY_SIGNAL && only.value == 0.0 + ), + "a failed or stale provider keeps a ready=0 row, so an alert sees it" + ); + assert!( + published_signals(None, ready(1.0), None, false).is_none(), + "a waiting or unconfigured provider publishes nothing" + ); + } + #[test] fn capacity_is_max_running_per_endpoint_times_ready_endpoints() { let mut provider = provider_with_status(&serde_json::json!({})); @@ -4122,12 +4154,12 @@ mod tests { )] fn the_status_column_is_written_with_the_condition_and_backfilled_once() { let down = readiness::Verdict { - reason: readiness::Reason::NoReadyEndpoints, + reason: readiness::Reason::NoEndpointsReady, message: "0 ready endpoints".to_owned(), }; // The condition already says so, but STATE was never written: backfill it. let unshown = provider_with_status(&serde_json::json!({ - "conditions": [ready_condition_json("False", "NoReadyEndpoints")] + "conditions": [ready_condition_json("False", "NoEndpointsReady")] })); let (_, patch) = ready_condition_patch(&unshown, &down).expect("state is backfilled"); assert_eq!(patch["status"]["state"], "NotReady"); @@ -4137,7 +4169,7 @@ mod tests { ); // Both already current: nothing to write. let shown = provider_with_status(&serde_json::json!({ - "conditions": [ready_condition_json("False", "NoReadyEndpoints")], + "conditions": [ready_condition_json("False", "NoEndpointsReady")], "state": "NotReady" })); assert!(ready_condition_patch(&shown, &down).is_none()); diff --git a/operator/src/metrics.rs b/operator/src/metrics.rs index fed4efb4..af05f8a8 100644 --- a/operator/src/metrics.rs +++ b/operator/src/metrics.rs @@ -41,6 +41,10 @@ static REGISTRY: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_SIGNALS_REFUSED.clone())) .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(PROVIDER_SCRAPES.clone())) + .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(PROVIDER_LAST_SCRAPE_SUCCESS.clone())) + .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_DURATION.clone())) .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_SLOW.clone())) @@ -127,6 +131,28 @@ static PEER_POLL_TOTAL: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()) }); +/// Provider metrics scrapes by provider and result: `success`, `no_series`, or a failure class +/// (`timeout`, `unauthorized`, `tls`, `dns`, `connect`, `http`, `body_cap`, `parse`, `config`). +static PROVIDER_SCRAPES: LazyLock = LazyLock::new(|| { + IntCounterVec::new( + Opts::new("grid_provider_scrape_total", "Provider metrics scrapes by result"), + &["grid_provider", "result"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + +/// When each provider's metrics last scraped with its ready-endpoint series, Unix seconds. +static PROVIDER_LAST_SCRAPE_SUCCESS: LazyLock = LazyLock::new(|| { + IntGaugeVec::new( + Opts::new( + "grid_provider_last_scrape_success_timestamp_seconds", + "Unix time of the provider's last scrape with its ready-endpoint series", + ), + &["grid_provider"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + /// Peer observations refused at ingest, by peer and reason. static PEER_SIGNALS_REFUSED: LazyLock = LazyLock::new(|| { IntCounterVec::new( @@ -438,6 +464,39 @@ pub(crate) fn record_peer_poll(peer: &str, outcome: &str, duration: Duration, by } } +/// Count one scrape of `provider` with `result`. +pub(crate) fn record_provider_scrape(provider: &str, result: &str) { + PROVIDER_SCRAPES.with_label_values(&[provider, result]).inc(); +} + +/// Record that `provider` last scraped with its ready-endpoint series at `at`. +pub(crate) fn set_provider_last_scrape_success(provider: &str, at: std::time::SystemTime) { + let secs = at.duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs(); + PROVIDER_LAST_SCRAPE_SUCCESS + .with_label_values(&[provider]) + .set(secs.try_into().unwrap_or(i64::MAX)); +} + +/// Drop `provider`'s scrape series once it is gone, so they do not outlive it. +pub(crate) fn forget_provider_scrapes(provider: &str) { + let _absent = PROVIDER_LAST_SCRAPE_SUCCESS.remove_label_values(&[provider]); + for result in [ + "success", + "no_series", + "timeout", + "unauthorized", + "tls", + "dns", + "connect", + "http", + "body_cap", + "parse", + "config", + ] { + let _absent_result = PROVIDER_SCRAPES.remove_label_values(&[provider, result]); + } +} + /// Count a peer observation this hub refused: `name`, `provider`, `value`, or `provider_cap`. pub(crate) fn record_peer_signal_refused(peer: &str, reason: &str) { PEER_SIGNALS_REFUSED.with_label_values(&[peer, reason]).inc(); @@ -772,6 +831,24 @@ mod tests { ); } + #[test] + fn provider_scrapes_are_counted_by_result_and_forgotten_with_the_provider() { + let provider = "scrape-count-test"; + record_provider_scrape(provider, "success"); + record_provider_scrape(provider, "no_series"); + record_provider_scrape(provider, "no_series"); + set_provider_last_scrape_success(provider, std::time::UNIX_EPOCH + Duration::from_secs(42)); + assert_eq!(PROVIDER_SCRAPES.with_label_values(&[provider, "success"]).get(), 1); + assert_eq!(PROVIDER_SCRAPES.with_label_values(&[provider, "no_series"]).get(), 2); + assert_eq!(PROVIDER_LAST_SCRAPE_SUCCESS.with_label_values(&[provider]).get(), 42); + forget_provider_scrapes(provider); + assert!( + PROVIDER_SCRAPES.remove_label_values(&[provider, "success"]).is_err(), + "forgotten with the provider" + ); + assert!(PROVIDER_LAST_SCRAPE_SUCCESS.remove_label_values(&[provider]).is_err()); + } + #[test] #[expect(clippy::too_many_lines, reason = "one store across two scrapes")] fn provider_signals_drop_a_series_the_store_no_longer_holds() { diff --git a/operator/src/metrics_scraper.rs b/operator/src/metrics_scraper.rs index 8fb2e148..2772e506 100644 --- a/operator/src/metrics_scraper.rs +++ b/operator/src/metrics_scraper.rs @@ -93,6 +93,58 @@ pub enum MetricsScrapeError { /// A credential would have gone over plain HTTP without `allowPlaintext`. #[error("refusing to send a metrics credential over plain http: {0}")] PlaintextCredential(String), + /// The body passed the 1 MiB response limit. + #[error("metrics response body exceeds {0} byte limit")] + BodyTooLarge(usize), +} + +impl MetricsScrapeError { + /// The bounded class of this failure, for the provider's condition and scrape counters. + #[must_use] + pub(crate) fn class(&self) -> crate::readiness::ScrapeClass { + use crate::readiness::ScrapeClass; + match self { + Self::Timeout(_) => ScrapeClass::Timeout, + Self::NonOkStatus { status: 401 | 403, .. } => ScrapeClass::Unauthorized, + Self::NonOkStatus { .. } => ScrapeClass::Http, + Self::Transport(source) => transport_class(source.as_ref()), + Self::Encoding(_) => ScrapeClass::Parse, + Self::TlsMaterial(_) => ScrapeClass::Tls, + Self::BodyTooLarge(_) => ScrapeClass::BodyCap, + Self::InvalidUrl(_) | Self::HttpWithTls(_) | Self::Credential(_) | Self::PlaintextCredential(_) => { + ScrapeClass::Config + }, + } + } +} + +/// TLS when the chain holds a TLS error, DNS when resolution failed, timeout when the socket +/// timed out, else a connect failure. +fn transport_class(source: &(dyn std::error::Error + 'static)) -> crate::readiness::ScrapeClass { + use crate::readiness::ScrapeClass; + let mut cause = Some(source); + while let Some(error) = cause { + // io::Error's source skips the error it wraps, which is where a TLS failure sits. + let wrapped = error + .downcast_ref::() + .and_then(std::io::Error::get_ref) + .is_some_and(|inner| crate::resources::tls_backend::is_tls_error(inner)); + if wrapped || crate::resources::tls_backend::is_tls_error(error) { + return ScrapeClass::Tls; + } + let text = error.to_string(); + if text.contains("dns error") || text.contains("failed to lookup address") { + return ScrapeClass::Dns; + } + if error + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::TimedOut) + { + return ScrapeClass::Timeout; + } + cause = error.source(); + } + ScrapeClass::Connect } // --------------------------------------------------------------------------- @@ -299,9 +351,7 @@ async fn read_response( .await .map_err(|e| { if e.downcast_ref::().is_some() { - MetricsScrapeError::Transport( - format!("metrics response body exceeds {MAX_RESPONSE_BODY_BYTES} byte limit").into(), - ) + MetricsScrapeError::BodyTooLarge(MAX_RESPONSE_BODY_BYTES) } else { MetricsScrapeError::Transport(e) } @@ -327,6 +377,74 @@ mod tests { use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use super::*; + + #[test] + #[cfg(not(feature = "fips"))] + fn a_tls_failure_wrapped_in_an_io_error_classes_as_tls() { + let tls: Box = Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + rustls::Error::General("bad certificate".to_owned()), + )); + assert_eq!( + MetricsScrapeError::Transport(tls).class(), + crate::readiness::ScrapeClass::Tls + ); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one table of failures and their classes")] + fn a_scrape_failure_has_a_bounded_class() { + use crate::readiness::ScrapeClass; + let status = |status| MetricsScrapeError::NonOkStatus { + status, + url: "https://epp:9090/metrics".to_owned(), + }; + let transport = |error: Box| MetricsScrapeError::Transport(error); + let cases = [ + ( + "timeout", + MetricsScrapeError::Timeout(Duration::from_secs(2)), + ScrapeClass::Timeout, + ), + ("401", status(401), ScrapeClass::Unauthorized), + ("403", status(403), ScrapeClass::Unauthorized), + ("500", status(500), ScrapeClass::Http), + ("404", status(404), ScrapeClass::Http), + ( + "dns", + transport("dns error: failed to lookup address information".into()), + ScrapeClass::Dns, + ), + ( + "refused", + transport(Box::new(std::io::Error::from(std::io::ErrorKind::ConnectionRefused))), + ScrapeClass::Connect, + ), + ( + "socket timeout", + transport(Box::new(std::io::Error::from(std::io::ErrorKind::TimedOut))), + ScrapeClass::Timeout, + ), + ( + "body cap", + MetricsScrapeError::BodyTooLarge(MAX_RESPONSE_BODY_BYTES), + ScrapeClass::BodyCap, + ), + ( + "tls material", + MetricsScrapeError::TlsMaterial("no key".to_owned()), + ScrapeClass::Tls, + ), + ( + "plaintext", + MetricsScrapeError::PlaintextCredential("http://epp".to_owned()), + ScrapeClass::Config, + ), + ]; + for (label, error, class) in cases { + assert_eq!(error.class(), class, "{label}"); + } + } use crate::resources::tls_backend::{MAX_CA_PEM_BYTES, MAX_CLIENT_CERT_PEM_BYTES, MAX_CLIENT_KEY_PEM_BYTES}; /// Start a local HTTP server on a random port and return the URL. diff --git a/operator/src/readiness.rs b/operator/src/readiness.rs index debb23b4..7bea43ff 100644 --- a/operator/src/readiness.rs +++ b/operator/src/readiness.rs @@ -56,16 +56,26 @@ const EPP_AVERAGES: [[&str; 2]; 2] = [ pub enum Reason { /// Scraped recently with at least one ready endpoint. Ready, - /// The latest scrape counted zero ready endpoints. - NoReadyEndpoints, - /// No successful scrape within the staleness window. + /// The latest scrapes counted zero ready endpoints. + NoEndpointsReady, + /// The scrape answered, but without the pool's ready-endpoint series. + NoLivenessCheck, + /// No successful scrape within the staleness window, and no attempt failed. MetricsStale, - /// No successful scrape within the window, and the latest attempt failed. - MetricsUnreachable, + /// No successful scrape within the window, and the latest attempt timed out. + ScrapeTimedOut, + /// No successful scrape within the window, and the latest attempt was refused (401 or 403). + ScrapeUnauthorized, + /// No successful scrape within the window, and the latest attempt failed TLS. + TlsHandshakeFailed, + /// No successful scrape within the window, and the latest attempt failed otherwise. + ScrapeFailed, /// The provider itself is `Unavailable`. ProviderUnavailable, /// The provider declares no metrics, so readiness is unknown. MetricsNotConfigured, + /// Attempted, with no scrape succeeding yet inside the grace window. + AwaitingFirstScrape, } impl Reason { @@ -74,11 +84,16 @@ impl Reason { pub const fn as_str(self) -> &'static str { match self { Self::Ready => "Ready", - Self::NoReadyEndpoints => "NoReadyEndpoints", + Self::NoEndpointsReady => "NoEndpointsReady", + Self::NoLivenessCheck => "NoLivenessCheck", Self::MetricsStale => "MetricsStale", - Self::MetricsUnreachable => "MetricsUnreachable", + Self::ScrapeTimedOut => "ScrapeTimedOut", + Self::ScrapeUnauthorized => "ScrapeUnauthorized", + Self::TlsHandshakeFailed => "TLSHandshakeFailed", + Self::ScrapeFailed => "ScrapeFailed", Self::ProviderUnavailable => "ProviderUnavailable", Self::MetricsNotConfigured => "MetricsNotConfigured", + Self::AwaitingFirstScrape => "AwaitingFirstScrape", } } @@ -87,10 +102,15 @@ impl Reason { pub const fn status(self) -> &'static str { match self { Self::Ready => "True", - Self::MetricsNotConfigured => "Unknown", - Self::NoReadyEndpoints | Self::MetricsStale | Self::MetricsUnreachable | Self::ProviderUnavailable => { - "False" - }, + Self::MetricsNotConfigured | Self::AwaitingFirstScrape => "Unknown", + Self::NoEndpointsReady + | Self::NoLivenessCheck + | Self::MetricsStale + | Self::ScrapeTimedOut + | Self::ScrapeUnauthorized + | Self::TlsHandshakeFailed + | Self::ScrapeFailed + | Self::ProviderUnavailable => "False", } } @@ -99,23 +119,79 @@ impl Reason { pub const fn display(self) -> &'static str { match self { Self::Ready => "Ready", - Self::MetricsNotConfigured => "Unknown", - Self::NoReadyEndpoints | Self::MetricsStale | Self::MetricsUnreachable | Self::ProviderUnavailable => { - "NotReady" - }, + Self::MetricsNotConfigured | Self::AwaitingFirstScrape => "Unknown", + Self::NoEndpointsReady + | Self::NoLivenessCheck + | Self::MetricsStale + | Self::ScrapeTimedOut + | Self::ScrapeUnauthorized + | Self::TlsHandshakeFailed + | Self::ScrapeFailed + | Self::ProviderUnavailable => "NotReady", } } /// Whether the provider is known not to serve: unknown counts as ready. #[must_use] pub const fn excludes(self) -> bool { - matches!( + !matches!( self, - Self::NoReadyEndpoints | Self::MetricsStale | Self::MetricsUnreachable | Self::ProviderUnavailable + Self::Ready | Self::MetricsNotConfigured | Self::AwaitingFirstScrape ) } } +/// Why one scrape failed, bounded so it can label a metric. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScrapeClass { + /// The request timed out. + Timeout, + /// The endpoint answered 401 or 403. + Unauthorized, + /// TLS failed: a handshake, a certificate, or the TLS material. + Tls, + /// The endpoint's name did not resolve. + Dns, + /// The connection failed. + Connect, + /// The endpoint answered with another non-2xx status. + Http, + /// The body passed the size limit. + BodyCap, + /// The body could not be read as text. + Parse, + /// The scrape could not be built: the URL, the credential, or plaintext refused. + Config, +} + +impl ScrapeClass { + /// The class as a metric label and message word. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::Unauthorized => "unauthorized", + Self::Tls => "tls", + Self::Dns => "dns", + Self::Connect => "connect", + Self::Http => "http", + Self::BodyCap => "body_cap", + Self::Parse => "parse", + Self::Config => "config", + } + } + + /// The `Ready` reason when this was the latest failure and nothing succeeded since. + const fn reason(self) -> Reason { + match self { + Self::Timeout => Reason::ScrapeTimedOut, + Self::Unauthorized => Reason::ScrapeUnauthorized, + Self::Tls => Reason::TlsHandshakeFailed, + Self::Dns | Self::Connect | Self::Http | Self::BodyCap | Self::Parse | Self::Config => Reason::ScrapeFailed, + } + } +} + /// What the signals loop last saw from one provider. #[derive(Clone, Debug, Default)] struct Probe { @@ -127,8 +203,10 @@ struct Probe { ready_endpoints: Option, /// Signals from the latest success, until the signals loop publishes them once. unpublished: Option>, - /// Whether the latest attempt failed. - failing: bool, + /// The latest attempt's failure, `None` when it succeeded. + failure: Option, + /// What the latest success lacked, when it exposed no ready-endpoint count. + missing: Option, /// Consecutive successful scrapes that read zero ready endpoints. zero_streak: u32, /// Consecutive successful scrapes that read some ready endpoints. @@ -180,12 +258,14 @@ impl ReadinessStore { .record(snapshot, now) } - /// Record a successful scrape at `now`. + /// Record a successful scrape at `now`, with `missing` naming the series it lacked. #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] + #[expect(clippy::too_many_arguments, reason = "one scrape's count, gap, signals, and time")] pub(crate) fn record_success( &self, key: &str, ready_endpoints: Option, + missing: Option, observations: Vec, now: Instant, ) { @@ -213,33 +293,40 @@ impl ReadinessStore { probe.last_good = Some(now); probe.ready_endpoints = ready_endpoints; probe.unpublished = Some(observations); - probe.failing = false; + probe.failure = None; + probe.missing = ready_endpoints.is_none().then_some(missing).flatten(); } /// Record a failed scrape at `now`, keeping what the last good one saw. #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] - pub(crate) fn record_failure(&self, key: &str, now: Instant) { + pub(crate) fn record_failure(&self, key: &str, class: ScrapeClass, now: Instant) { let mut probes = self.probes(); let probe = probes.entry(key.to_owned()).or_default(); probe.first_attempt.get_or_insert(now); - probe.failing = true; + probe.failure = Some(class); } /// The verdict for `key` at `now`. /// - /// `None` until the provider has been attempted, and through the first - /// `stale_after` while no scrape has succeeded yet, so a starting operator - /// publishes nothing rather than a guess. + /// `AwaitingFirstScrape`, which is `Unknown` and publishes nothing, until the + /// provider has been attempted and through the first `stale_after` while no + /// scrape has succeeded yet, so a starting operator states that it is waiting. #[expect( clippy::significant_drop_tightening, reason = "the guard covers one small evaluation" )] pub(crate) fn verdict(&self, key: &str, unavailable: bool, stale_after: Duration, now: Instant) -> Option { + let awaiting = Verdict { + reason: Reason::AwaitingFirstScrape, + message: "awaiting the first metrics scrape".to_owned(), + }; let probes = self.probes(); - let probe = probes.get(key)?; + let Some(probe) = probes.get(key) else { + return (!unavailable).then_some(awaiting); + }; let within = |at: Instant| now.saturating_duration_since(at) <= stale_after; if probe.last_good.is_none() && !unavailable && probe.first_attempt.is_some_and(within) { - return None; + return Some(awaiting); } Some(evaluate( probe, @@ -261,7 +348,13 @@ impl ReadinessStore { /// Forget providers not in `keep`. pub(crate) fn retain(&self, keep: &std::collections::BTreeSet) { - self.probes().retain(|key, _| keep.contains(key)); + self.probes().retain(|key, _| { + let kept = keep.contains(key); + if !kept && let Some((_, identity)) = key.split_once('/') { + crate::metrics::forget_provider_scrapes(identity); + } + kept + }); } } @@ -274,14 +367,19 @@ fn evaluate(probe: &Probe, fresh: bool, unavailable: bool, stale_after: Duration }; } if !fresh { - let reason = if probe.failing { - Reason::MetricsUnreachable - } else { - Reason::MetricsStale - }; - return Verdict { - reason, - message: format!("no successful metrics scrape in the last {}s", stale_after.as_secs()), + let window = stale_after.as_secs(); + return match probe.failure { + Some(class) => Verdict { + reason: class.reason(), + message: format!( + "no successful metrics scrape in the last {window}s; the latest failed: {}", + class.as_str() + ), + }, + None => Verdict { + reason: Reason::MetricsStale, + message: format!("no successful metrics scrape in the last {window}s"), + }, }; } from_ready_endpoints(probe) @@ -296,14 +394,23 @@ fn from_ready_endpoints(probe: &Probe) -> Verdict { _ => format!("0 ready endpoints for {STREAK} or more scrapes"), }; return Verdict { - reason: Reason::NoReadyEndpoints, + reason: Reason::NoEndpointsReady, message, }; } - let message = match count { - Some(count) if count < 1.0 => "0 ready endpoints in the latest scrape only".to_owned(), - Some(count) => format!("{count} ready endpoints"), - None => "metrics reachable, no ready-endpoint count exposed".to_owned(), + let Some(count) = count else { + return Verdict { + reason: Reason::NoLivenessCheck, + message: probe + .missing + .clone() + .unwrap_or_else(|| "metrics reachable, but no ready-endpoint series".to_owned()), + }; + }; + let message = if count < 1.0 { + "0 ready endpoints in the latest scrape only".to_owned() + } else { + format!("{count} ready endpoints") }; Verdict { reason: Reason::Ready, @@ -504,31 +611,43 @@ mod tests { fn verdict_follows_the_latest_scrape_and_staleness() { let store = ReadinessStore::default(); let start = Instant::now(); - assert_eq!(reason(&store, start), None, "never attempted"); + assert_eq!( + reason(&store, start), + Some(Reason::AwaitingFirstScrape), + "never attempted" + ); - store.record_success("p", Some(2.0), vec![sample("q", "p", 1.0)], start); + store.record_success("p", Some(2.0), None, vec![sample("q", "p", 1.0)], start); assert_eq!(reason(&store, start), Some(Reason::Ready)); assert_eq!(store.take_fresh("p").map(|held| held.len()), Some(1)); assert_eq!(store.take_fresh("p"), None, "fresh signals are published once"); - store.record_success("p", None, Vec::new(), start); + let missing = "metrics reachable, but no llm_d_epp_ready_endpoints series for poolName qwen3"; + store.record_success("p", None, Some(missing.to_owned()), Vec::new(), start); + let verdict = store.verdict("p", false, STALE, start).expect("judged"); assert_eq!( - reason(&store, start), - Some(Reason::Ready), - "a scrape without the count is not read as zero" + (verdict.reason, verdict.message.as_str()), + (Reason::NoLivenessCheck, missing), + "a 200 without the pool's series is not ready, and says which series" ); + assert!(verdict.reason.excludes()); assert_eq!(store.take_fresh("p").map(|held| held.len()), Some(0)); - store.record_failure("p", start + STALE); + store.record_failure("p", ScrapeClass::Connect, start + STALE); assert_eq!( reason(&store, start + STALE), - Some(Reason::Ready), + Some(Reason::NoLivenessCheck), "a failure within the window keeps the last verdict" ); assert_eq!(store.take_fresh("p"), None, "a failure publishes no held load"); - assert_eq!( - reason(&store, start + STALE + Duration::from_secs(1)), - Some(Reason::MetricsUnreachable) + let stale = store + .verdict("p", false, STALE, start + STALE + Duration::from_secs(1)) + .expect("judged"); + assert_eq!(stale.reason, Reason::ScrapeFailed); + assert!( + stale.message.ends_with("the latest failed: connect"), + "{}", + stale.message ); } @@ -537,22 +656,22 @@ mod tests { let store = ReadinessStore::default(); let now = Instant::now(); let scrape = |count| { - store.record_success("p", Some(count), Vec::new(), now); + store.record_success("p", Some(count), None, Vec::new(), now); reason(&store, now) }; assert_eq!(scrape(0.0), Some(Reason::Ready), "one zero reading is not enough"); - assert_eq!(scrape(0.0), Some(Reason::NoReadyEndpoints)); + assert_eq!(scrape(0.0), Some(Reason::NoEndpointsReady)); assert_eq!( scrape(1.0), - Some(Reason::NoReadyEndpoints), + Some(Reason::NoEndpointsReady), "one ready reading is not enough" ); assert_eq!( scrape(0.0), - Some(Reason::NoReadyEndpoints), + Some(Reason::NoEndpointsReady), "a relapse restarts the count" ); - assert_eq!(scrape(1.0), Some(Reason::NoReadyEndpoints)); + assert_eq!(scrape(1.0), Some(Reason::NoEndpointsReady)); assert_eq!(scrape(1.0), Some(Reason::Ready)); } @@ -560,11 +679,50 @@ mod tests { fn a_first_scrape_that_fails_gets_the_staleness_window_before_a_verdict() { let store = ReadinessStore::default(); let start = Instant::now(); - store.record_failure("p", start); - assert_eq!(reason(&store, start + STALE), None, "grace, not a guess"); + store.record_failure("p", ScrapeClass::Timeout, start); + assert_eq!( + reason(&store, start + STALE), + Some(Reason::AwaitingFirstScrape), + "grace, not a guess" + ); assert_eq!( reason(&store, start + STALE + Duration::from_secs(1)), - Some(Reason::MetricsUnreachable) + Some(Reason::ScrapeTimedOut) + ); + } + + #[test] + fn a_failed_scrape_names_its_class() { + let cases = [ + (ScrapeClass::Timeout, Reason::ScrapeTimedOut), + (ScrapeClass::Unauthorized, Reason::ScrapeUnauthorized), + (ScrapeClass::Tls, Reason::TlsHandshakeFailed), + (ScrapeClass::Dns, Reason::ScrapeFailed), + (ScrapeClass::Connect, Reason::ScrapeFailed), + (ScrapeClass::Http, Reason::ScrapeFailed), + (ScrapeClass::BodyCap, Reason::ScrapeFailed), + (ScrapeClass::Parse, Reason::ScrapeFailed), + (ScrapeClass::Config, Reason::ScrapeFailed), + ]; + let start = Instant::now(); + for (class, expected) in cases { + let store = ReadinessStore::default(); + store.record_failure("p", class, start); + let verdict = store + .verdict("p", false, STALE, start + STALE + Duration::from_secs(1)) + .expect("judged"); + assert_eq!(verdict.reason, expected, "{class:?}"); + assert!( + verdict.message.ends_with(class.as_str()), + "{class:?}: {}", + verdict.message + ); + assert!(verdict.reason.excludes(), "{class:?}"); + } + assert_eq!( + Reason::TlsHandshakeFailed.as_str(), + "TLSHandshakeFailed", + "the contract's spelling" ); } @@ -573,15 +731,15 @@ mod tests { let store = ReadinessStore::default(); let now = Instant::now(); for _ in 0..STREAK { - store.record_success(&key("east", "p"), Some(0.0), Vec::new(), now); + store.record_success(&key("east", "p"), Some(0.0), None, Vec::new(), now); } - store.record_success(&key("west", "p"), Some(3.0), Vec::new(), now); + store.record_success(&key("west", "p"), Some(3.0), None, Vec::new(), now); let reason_in = |network| { store .verdict(&key(network, "p"), false, STALE, now) .map(|verdict| verdict.reason) }; - assert_eq!(reason_in("east"), Some(Reason::NoReadyEndpoints)); + assert_eq!(reason_in("east"), Some(Reason::NoEndpointsReady)); assert_eq!(reason_in("west"), Some(Reason::Ready)); } @@ -589,7 +747,7 @@ mod tests { fn an_unavailable_provider_is_not_ready_whatever_its_metrics_say() { let store = ReadinessStore::default(); let now = Instant::now(); - store.record_success("p", Some(3.0), Vec::new(), now); + store.record_success("p", Some(3.0), None, Vec::new(), now); let verdict = store.verdict("p", true, STALE, now).unwrap(); assert_eq!(verdict.reason, Reason::ProviderUnavailable); assert!(verdict.reason.excludes()); @@ -599,10 +757,12 @@ mod tests { fn the_status_column_reads_like_a_node() { assert_eq!(Reason::Ready.display(), "Ready"); assert_eq!(Reason::MetricsNotConfigured.display(), "Unknown"); + assert_eq!(Reason::AwaitingFirstScrape.display(), "Unknown"); for reason in [ - Reason::NoReadyEndpoints, + Reason::NoEndpointsReady, + Reason::NoLivenessCheck, Reason::MetricsStale, - Reason::MetricsUnreachable, + Reason::ScrapeFailed, Reason::ProviderUnavailable, ] { assert_eq!(reason.display(), "NotReady", "{reason:?}"); @@ -637,7 +797,7 @@ mod tests { ); let down = Verdict { - reason: Reason::NoReadyEndpoints, + reason: Reason::NoEndpointsReady, message: "0 ready endpoints".to_owned(), }; let second = ready_condition(std::slice::from_ref(&first), &down, "t2", Some(3)).unwrap(); @@ -647,7 +807,7 @@ mod tests { ); let unreachable = Verdict { - reason: Reason::MetricsUnreachable, + reason: Reason::ScrapeFailed, message: "no scrape".to_owned(), }; let third = ready_condition(std::slice::from_ref(&second), &unreachable, "t3", Some(3)).unwrap(); @@ -662,10 +822,16 @@ mod tests { assert_eq!(Reason::Ready.status(), "True"); assert_eq!(Reason::MetricsNotConfigured.status(), "Unknown"); assert!(!Reason::MetricsNotConfigured.excludes(), "unknown is not down"); + assert_eq!(Reason::AwaitingFirstScrape.status(), "Unknown"); + assert!(!Reason::AwaitingFirstScrape.excludes(), "waiting is not down"); for reason in [ - Reason::NoReadyEndpoints, + Reason::NoEndpointsReady, Reason::MetricsStale, - Reason::MetricsUnreachable, + Reason::NoLivenessCheck, + Reason::ScrapeTimedOut, + Reason::ScrapeUnauthorized, + Reason::TlsHandshakeFailed, + Reason::ScrapeFailed, Reason::ProviderUnavailable, ] { assert_eq!(reason.status(), "False"); diff --git a/operator/src/resources/provider_metrics.rs b/operator/src/resources/provider_metrics.rs index 3fed9a60..0bd4eb59 100644 --- a/operator/src/resources/provider_metrics.rs +++ b/operator/src/resources/provider_metrics.rs @@ -130,8 +130,11 @@ pub(crate) async fn collect_provider_signals( }; let key = crate::readiness::key(network_name, plan.identity); match scrape_provider_signals(provider, &plan, client).await { - Some(text) => record_scrape(readiness, &key, provider, &plan, &text), - None => readiness.record_failure(&key, Instant::now()), + Ok(text) => record_scrape(readiness, &key, provider, &plan, &text), + Err(class) => { + crate::metrics::record_provider_scrape(plan.identity, class.as_str()); + readiness.record_failure(&key, class, Instant::now()); + }, } } } @@ -151,6 +154,7 @@ fn record_scrape( .as_ref() .and_then(|mc| mc.pool_name.as_deref()); let ready = crate::readiness::ready_endpoints(&parsed, plan.ready_names(), pool); + let missing = count_scrape(plan, pool, ready); let in_flight = in_flight_observation(&parsed, ready, pool, plan.identity); let latency = readiness.record_latency(key, &parsed, pool, Instant::now()); let observations = parsed @@ -165,7 +169,23 @@ fn record_scrape( .chain(in_flight) .chain(latency) .collect(); - readiness.record_success(key, ready, observations, Instant::now()); + readiness.record_success(key, ready, missing, observations, Instant::now()); +} + +/// Count a scrape that answered: `success` with the pool's ready-endpoint series, else +/// `no_series`, returned as the message naming what was missing. +fn count_scrape(plan: &SignalScrapePlan<'_>, pool: Option<&str>, ready: Option) -> Option { + if ready.is_some() { + crate::metrics::record_provider_scrape(plan.identity, "success"); + crate::metrics::set_provider_last_scrape_success(plan.identity, std::time::SystemTime::now()); + return None; + } + crate::metrics::record_provider_scrape(plan.identity, "no_series"); + let series = plan.ready_names().join(" or "); + Some(match pool { + Some(pool) => format!("metrics reachable, but no {series} series for poolName {pool}"), + None => format!("metrics reachable, but no {series} series"), + }) } /// The resolved `grid_provider_in_flight_requests` sample, logging which source it came from. @@ -195,25 +215,29 @@ async fn scrape_provider_signals( provider: &InferenceProvider, plan: &SignalScrapePlan<'_>, client: Option<&kube::Client>, -) -> Option { - let mc = provider.spec.metrics_config.as_ref()?; +) -> Result { + let mc = provider + .spec + .metrics_config + .as_ref() + .ok_or(crate::readiness::ScrapeClass::Config)?; let identity = plan.identity; let tls_config = match resolve_tls_config(mc.tls.as_ref(), client, identity).await { Ok(cfg) => cfg, Err((_reason, e)) => { if mc.tls.is_some() { - tracing::warn!(provider = identity, error = %e, "signals: provider metrics TLS unavailable; not scraping in plaintext"); + tracing::debug!(provider = identity, error = %e, "signals: provider metrics TLS unavailable; not scraping in plaintext"); } - return None; + return Err(crate::readiness::ScrapeClass::Tls); }, }; let timeout = parse_metrics_timeout(&mc.timeout); scrape_metrics(&plan.url, timeout, tls_config, mc.auth.as_ref().zip(client)) .await - .inspect_err(|e| { + .map_err(|e| { tracing::debug!(provider = identity, error = %e, "signals: provider scrape failed"); + e.class() }) - .ok() } /// What to scrape for one provider and which of its series to republish. @@ -721,6 +745,7 @@ pub(crate) fn classify_scrape_error(err: &metrics_scraper::MetricsScrapeError) - }, metrics_scraper::MetricsScrapeError::Credential(_) | metrics_scraper::MetricsScrapeError::PlaintextCredential(_) => "MetricsCredentialUnavailable", + metrics_scraper::MetricsScrapeError::BodyTooLarge(_) => "MetricsBodyTooLarge", metrics_scraper::MetricsScrapeError::InvalidUrl(_) | metrics_scraper::MetricsScrapeError::NonOkStatus { .. } | metrics_scraper::MetricsScrapeError::Encoding(_) => "MetricsScrapeError", diff --git a/operator/src/signals.rs b/operator/src/signals.rs index 39427f6c..fa9110ae 100644 --- a/operator/src/signals.rs +++ b/operator/src/signals.rs @@ -829,6 +829,7 @@ fn classify(error: &MetricsScrapeError) -> PollOutcome { } }, MetricsScrapeError::Encoding(_) => PollOutcome::Encoding, + MetricsScrapeError::BodyTooLarge(_) => PollOutcome::Transport, MetricsScrapeError::InvalidUrl(_) | MetricsScrapeError::HttpWithTls(_) | MetricsScrapeError::TlsMaterial(_) From 2802be4a0dead30cb5d23d9fe2c2b1c0de217b44 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 14:29:28 -0400 Subject: [PATCH 07/10] fix(operator): place providers at their own site and refuse bad serving sites A provider with an empty site selector is placed at its own site only. The serving config refuses candidates whose site is not a DNS-1123 label, warning per GridNetwork on change, and derives the gateway's load window from the scrape, poll, and poll timeout plus one poll of slack, 17s at a 5s scrape. Signed-off-by: Sam Batschelet --- .../testdata/serving-config.json | 2 +- operator/src/controller/grid_network.rs | 12 ++ operator/src/controller/inference_provider.rs | 1 - operator/src/metrics.rs | 1 - operator/src/resources/routing_overlay.rs | 133 +++++++++++++----- operator/src/resources/serving_config.rs | 104 ++++++++++++-- 6 files changed, 203 insertions(+), 50 deletions(-) diff --git a/gateway/ai-grid-filters/testdata/serving-config.json b/gateway/ai-grid-filters/testdata/serving-config.json index ad100ebf..40ae0a51 100644 --- a/gateway/ai-grid-filters/testdata/serving-config.json +++ b/gateway/ai-grid-filters/testdata/serving-config.json @@ -1,7 +1,7 @@ { "local_site": "site-a", "window_secs": 60, - "load_window_ms": 10000, + "load_window_ms": 17000, "candidates": [ { "kind": "inference_model", diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index bec88be5..5339a31d 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -99,6 +99,9 @@ pub struct OperatorCtx { /// after async DNS resolution and the SWIM channel announcement. pub(crate) last_seeds: std::sync::Mutex>>, + /// Sites each `GridNetwork`'s serving config last refused, for warning on change. + pub(crate) refused_sites: serving_config::RefusedSites, + /// Who may read the signals endpoint, keyed by presented-cert fingerprint. /// Set by reconcile from each `GridSite`'s trust pins. pub(crate) peer_identities: signals::PeerIdentities, @@ -278,6 +281,7 @@ impl OperatorCtx { metrics_cache: Mutex::new(provider_metrics::MetricsCache::new()), admission_memory: Mutex::new(provider_admission::AdmissionMemory::default()), last_seeds: std::sync::Mutex::new(HashMap::new()), + refused_sites: std::sync::Mutex::new(HashMap::new()), peer_identities: signals::PeerIdentities::new(), peers: signals::SignalStore::new(), signals: signals::SignalStore::new(), @@ -2084,6 +2088,10 @@ struct ServingSource<'src> { gate: &'src WriteGate, /// Peer addressing resolved at startup. settings: &'src PeerSettings, + /// How often this operator scrapes its providers. + scrape_interval: Duration, + /// Sites each network's serving config last refused. + refused: &'src serving_config::RefusedSites, } /// Build the serving source from membership, `None` outside poll mode. @@ -2115,6 +2123,8 @@ fn serving_source<'src>( pins, gate: &ctx.serving_writes, settings: &ctx.peer_settings, + scrape_interval: ctx.scrape_interval, + refused: &ctx.refused_sites, }) } @@ -2134,6 +2144,7 @@ fn render_serving_text( tls_mount, local_signals_addr: source.settings.local_signals_addr.as_deref(), pins: &source.pins, + scrape_interval: source.scrape_interval, }; let members = source.members.iter().map(|(site, endpoint)| (*site, endpoint.as_str())); serving_config::render(overlay, members, &inputs) @@ -2151,6 +2162,7 @@ async fn apply_serving_config( gw_ref: &GatewayRef, client: &Client, ) -> Result, OperatorError> { + serving_config::warn_refused_sites(overlay, network_name, source.refused); let Some(text) = render_serving_text(overlay, source, gw_ref)? else { tracing::debug!(gateway = %gw_ref.name, "serving config has no candidates; leaving any prior config"); return Ok(None); diff --git a/operator/src/controller/inference_provider.rs b/operator/src/controller/inference_provider.rs index dd2f6e4d..cc9a86aa 100644 --- a/operator/src/controller/inference_provider.rs +++ b/operator/src/controller/inference_provider.rs @@ -646,7 +646,6 @@ pub(crate) fn sites_matching_selector(provider: &InferenceProvider, sites: &[Gri names } - /// The sites hosting `provider`: this site alone when its selector is empty and this site is in /// the provider's network, as the routing overlay attributes it, else every site the selector matches. fn hosting_sites(provider: &InferenceProvider, sites: &[GridSite], local_site: Option<&str>) -> Vec { diff --git a/operator/src/metrics.rs b/operator/src/metrics.rs index af05f8a8..1ab36aee 100644 --- a/operator/src/metrics.rs +++ b/operator/src/metrics.rs @@ -113,7 +113,6 @@ static SWIM_PENDING_DROPS: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()) }); - // --------------------------------------------------------------------------- // Peer polling // diff --git a/operator/src/resources/routing_overlay.rs b/operator/src/resources/routing_overlay.rs index 1f357c49..43fc47a6 100644 --- a/operator/src/resources/routing_overlay.rs +++ b/operator/src/resources/routing_overlay.rs @@ -12,9 +12,9 @@ //! # Phase 1 / OP-01 semantics //! //! - [`GridSite`]s are used to resolve per-provider site membership via `spec.siteSelector.matchLabels`. An empty -//! selector matches all sites in the same [`GridNetwork`]. +//! selector means this site alone, when the network lists it. //! - Each `(model, site)` pair becomes one `RoutingCandidate`. -//! - `candidate.site` = the [`GridSite`] name (resolved via selector). +//! - `candidate.site` = the [`GridSite`]'s site id (resolved via selector), as peers are keyed. //! - `candidate.cluster` = `spec.routingClusterRef` when set, otherwise the [`InferenceProvider`] metadata name. The //! gateway uses this as the upstream cluster reference in its local routing configuration. //! - When no [`GridSite`]s are provided, the routing identity (`spec.routingClusterRef` or provider name) is used as @@ -48,6 +48,7 @@ use k8s_openapi::api::core::v1::ConfigMap; use serde::{Deserialize, Serialize}; use crate::{ + controller::grid_network::peer_site_key, crd::{ auth::{AccessPolicy, AuthStrategy}, grid_network::GridNetwork, @@ -1123,7 +1124,7 @@ fn assign_selection_groups(candidates: &mut [RoutingCandidate], policy: crate::c /// Only [`InferenceProvider`]s whose `spec.gridNetworkRef` matches /// `network.metadata.name` are included. Each provider's /// `spec.siteSelector.matchLabels` is matched against the supplied -/// `sites`; an empty selector matches all sites in the network. +/// `sites`; an empty selector means `local_site` alone. /// /// The `local_site` parameter identifies this gateway's own site. /// Praxis uses it to score candidates running on the local site higher @@ -1277,7 +1278,7 @@ pub fn render_routing_overlay_with_admission( .find(|site| site.metadata.name.as_deref() == Some(local_site) && site.spec.grid_network_ref == network_name) .and_then(|site| site.metadata.labels.as_ref()); - let mut candidates = collect_candidates(network_name, sites, providers, consumer_site_labels)?; + let mut candidates = collect_candidates(network_name, sites, providers, consumer_site_labels, local_site)?; for provider in remote_crdt_providers { let access_policy = crdt_access_policy_to_operator(&provider.access_policy); let access_result = evaluate_access_policy(&access_policy, consumer_site_labels); @@ -1462,6 +1463,7 @@ fn collect_candidates( sites: &[GridSite], providers: &[InferenceProvider], consumer_site_labels: Option<&BTreeMap>, + local_site: &str, ) -> Result, String> { // Pre-filter sites to those in this network. let network_sites: Vec<&GridSite> = sites @@ -1483,7 +1485,7 @@ fn collect_candidates( continue; } - let resolution = resolve_sites(provider, &network_sites); + let resolution = resolve_sites(provider, &network_sites, local_site); all.extend(candidates_from_provider(provider, &resolution)?); } Ok(all) @@ -1495,12 +1497,25 @@ fn collect_candidates( /// which enables the Phase 1 provider-name fallback. Returns /// [`SiteResolution::Known`] otherwise — with an empty `Vec` if the /// selector matched nothing, which suppresses candidate generation. -fn resolve_sites(provider: &InferenceProvider, network_sites: &[&GridSite]) -> SiteResolution { +/// +/// An empty selector means this site alone, when the inventory lists it, as the +/// provider controller attributes it. Sites are keyed as peers are, so a +/// discovered peer reads as its site id, not its object name. +fn resolve_sites(provider: &InferenceProvider, network_sites: &[&GridSite], local_site: &str) -> SiteResolution { if network_sites.is_empty() { return SiteResolution::Unavailable; } let selector = &provider.spec.site_selector.match_labels; + let key = |site: &&GridSite| peer_site_key(site).map(|(key, _)| key); + if selector.is_empty() { + let listed = network_sites.iter().filter_map(key).any(|site| site == local_site); + return SiteResolution::Known(if listed { + vec![local_site.to_owned()] + } else { + Vec::new() + }); + } let names: Vec = network_sites .iter() @@ -1510,7 +1525,7 @@ fn resolve_sites(provider: &InferenceProvider, network_sites: &[&GridSite]) -> S .iter() .all(|(k, v)| site_labels.is_some_and(|labels| labels.get(k).is_some_and(|sv| sv == v))) }) - .map(|site| site.metadata.name.clone().unwrap_or_else(|| "unknown-site".to_owned())) + .filter_map(key) .collect::>() .into_iter() .collect(); @@ -1990,9 +2005,19 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + /// A label every labeled test site carries, so a provider can be placed at all of them. + const PLACEMENT: (&str, &str) = ("test.grid/placement", "all"); + + /// `provider` placed at every labeled test site, as an explicit selector does. + fn placed_everywhere(mut provider: InferenceProvider) -> InferenceProvider { + provider.spec.site_selector.match_labels = BTreeMap::from([(PLACEMENT.0.to_owned(), PLACEMENT.1.to_owned())]); + provider + } + fn test_site_with_labels(name: &str, network: &str, labels: &[(&str, &str)]) -> GridSite { let labels_map: serde_json::Map = labels .iter() + .chain(std::iter::once(&PLACEMENT)) .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string()))) .collect(); serde_json::from_value(serde_json::json!({ @@ -3292,30 +3317,55 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn empty_selector_matches_all_sites_in_network() { - let network = test_network("net"); - let site_a = test_site("site-a", "net"); - let site_b = test_site("site-b", "net"); - let provider = test_provider("prov", "net", &["model"]); + fn an_empty_selector_places_a_provider_at_the_local_site_only() { + let render = |local: &str| { + render_routing_overlay( + &test_network("net"), + &[test_site("site-a", "net"), test_site("site-b", "net")], + &[test_provider("prov", "net", &["model"])], + &[], + local, + None, + None, + &scoring::ScoringWeights::default(), + ) + .map(|overlay| overlay.candidates.into_iter().map(|c| c.site).collect::>()) + }; + assert!( + matches!(render("site-a"), Ok(sites) if sites == ["site-a"]), + "not every site in the network" + ); + assert!( + matches!(render("absent"), Ok(sites) if sites.is_empty()), + "a local site the inventory does not list hosts nothing" + ); + } + + #[test] + fn a_selector_names_a_discovered_peer_by_its_site_id() { + let mut peer = test_site_with_labels("grid-retail", "net", &[("hw", "gpu")]); + peer.metadata.labels.get_or_insert_with(BTreeMap::new).insert( + crate::controller::grid_network::LABEL_AUTO_DISCOVERED.to_owned(), + "true".to_owned(), + ); + peer.metadata.annotations = Some(BTreeMap::from([( + crate::controller::grid_network::ANNOTATION_SITE_ID.to_owned(), + "retail".to_owned(), + )])); + let provider = test_provider_with_selector("prov", "net", &["model"], &[("hw", "gpu")]); let overlay = render_routing_overlay( - &network, - &[site_a, site_b], + &test_network("net"), + &[peer], &[provider], &[], - "test-site", + "hq", None, None, &scoring::ScoringWeights::default(), ) .unwrap_or_else(|_| std::process::abort()); - assert_eq!( - overlay.candidates.len(), - 2, - "empty selector should produce one candidate per site" - ); let sites: Vec<&str> = overlay.candidates.iter().map(|c| c.site.as_str()).collect(); - assert!(sites.contains(&"site-a"), "site-a must be in candidates"); - assert!(sites.contains(&"site-b"), "site-b must be in candidates"); + assert_eq!(sites, ["retail"], "keyed as peers are, not by object name"); } #[test] @@ -3412,7 +3462,7 @@ mod tests { &[site], &[p1, p2], &[], - "test-site", + "site-a", None, None, &scoring::ScoringWeights::default(), @@ -3438,7 +3488,7 @@ mod tests { &[site], &[provider], &[], - "test-site", + "site-a", None, None, &scoring::ScoringWeights::default(), @@ -3708,13 +3758,17 @@ mod tests { &[site_a, site_b], &[provider], &[], - "test-site", + "site-a", None, None, &scoring::ScoringWeights::default(), ) .unwrap_or_else(|_| std::process::abort()); - assert_eq!(overlay.candidates.len(), 2, "one candidate per matched site"); + assert_eq!( + overlay.candidates.len(), + 1, + "an empty selector places it at the local site" + ); assert!( overlay.candidates.iter().all(|c| !c.fresh), "every (model, site) candidate from a Degraded provider must have fresh=false" @@ -6316,7 +6370,7 @@ mod tests { let network = test_network("net"); let site_prod = test_site_with_labels("site-prod", "net", &[("env", "prod")]); let site_staging = test_site_with_labels("site-staging", "net", &[("env", "staging")]); - let provider = test_provider("unrestricted-prov", "net", &["model-a"]); + let provider = placed_everywhere(test_provider("unrestricted-prov", "net", &["model-a"])); // The provider has an empty siteSelector, so it should generate candidates // for ALL sites in the network when it appears in any overlay. @@ -6369,7 +6423,12 @@ mod tests { let network = test_network("net"); let site_prod = test_site_with_labels("site-prod", "net", &[("env", "prod"), ("region", "us-west")]); let site_staging = test_site_with_labels("site-staging", "net", &[("env", "staging")]); - let provider = test_provider_with_access_policy("prod-only-prov", "net", &["model-a"], &[("env", "prod")]); + let provider = placed_everywhere(test_provider_with_access_policy( + "prod-only-prov", + "net", + &["model-a"], + &[("env", "prod")], + )); // Consumer from prod site should get candidates // Since the provider has an empty siteSelector but restricted access policy, @@ -6490,7 +6549,7 @@ mod tests { let site = test_site_with_labels("known-site", "net", &[("env", "prod")]); let provider_restricted = test_provider_with_access_policy("restricted-prov", "net", &["model-a"], &[("env", "prod")]); - let provider_unrestricted = test_provider("unrestricted-prov", "net", &["model-b"]); + let provider_unrestricted = placed_everywhere(test_provider("unrestricted-prov", "net", &["model-b"])); // Consumer site not in the sites list (unknown identity) let overlay = render_routing_overlay( @@ -6525,15 +6584,19 @@ mod tests { let site_prod = test_site_with_labels("site-prod", "net", &[("env", "prod"), ("team", "platform")]); let site_staging = test_site_with_labels("site-staging", "net", &[("env", "staging"), ("team", "platform")]); - let provider_unrestricted = test_provider("unrestricted-prov", "net", &["model-general"]); - let provider_prod_only = - test_provider_with_access_policy("prod-only-prov", "net", &["model-prod"], &[("env", "prod")]); - let provider_platform_only = test_provider_with_access_policy( + let provider_unrestricted = placed_everywhere(test_provider("unrestricted-prov", "net", &["model-general"])); + let provider_prod_only = placed_everywhere(test_provider_with_access_policy( + "prod-only-prov", + "net", + &["model-prod"], + &[("env", "prod")], + )); + let provider_platform_only = placed_everywhere(test_provider_with_access_policy( "platform-only-prov", "net", &["model-platform"], &[("team", "platform")], - ); + )); // Prod consumer should get all three providers (each provider generates candidates for both sites) let overlay = render_routing_overlay( @@ -6714,7 +6777,7 @@ mod tests { let site = test_site_with_labels("known-site", "net", &[("env", "prod")]); let provider_restricted = test_provider_with_access_policy("local-restricted-prov", "net", &["model-a"], &[("env", "prod")]); - let provider_unrestricted = test_provider("local-unrestricted-prov", "net", &["model-b"]); + let provider_unrestricted = placed_everywhere(test_provider("local-unrestricted-prov", "net", &["model-b"])); // Consumer site not in the sites list (unknown identity) let overlay = render_routing_overlay( diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index 27e0e648..60e8d186 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -38,16 +38,21 @@ const WINDOW_SECS: u64 = 60; /// Peer poll interval, the operator's default local scrape interval. const PEER_INTERVAL_MS: u64 = 5_000; -/// Freshness window the gateway orders over: two polls, milliseconds. -/// -/// The gateway orders by the worst reading in the window, so a site drained by a -/// spike returns two polls after it clears instead of thirty seconds. -#[expect(clippy::cast_possible_wrap, reason = "a few seconds fits i64")] -const LOAD_WINDOW_MS: i64 = 2 * PEER_INTERVAL_MS as i64; - /// Peer connect and request timeout, milliseconds. const PEER_TIMEOUT_MS: u64 = 2_000; +/// The freshness window the gateway reads load over, milliseconds: the oldest a sample can +/// be when it arrives (a scrape, a poll, and a poll timeout) plus one poll of slack, so one +/// late poll does not turn a healthy site unknown. 17s at a 5s scrape. +fn load_window_ms(scrape: Duration) -> i64 { + let scrape_ms = u64::try_from(scrape.as_millis()).unwrap_or(u64::MAX); + let window = PEER_INTERVAL_MS + .saturating_add(scrape_ms) + .saturating_add(PEER_TIMEOUT_MS) + .saturating_add(PEER_INTERVAL_MS); + i64::try_from(window).unwrap_or(i64::MAX) +} + /// Gateway-side cap on candidates (`validate_candidates`). const MAX_CANDIDATES: usize = 1024; @@ -136,6 +141,8 @@ pub(crate) struct ServingInputs<'input> { pub(crate) local_signals_addr: Option<&'input str>, /// Declared leaf digests per remote site, empty outside pin trust. pub(crate) pins: &'input BTreeMap>, + /// How often operators scrape their providers, taken as every site's. + pub(crate) scrape_interval: Duration, } /// Render from `(site, signals endpoint)` members, `None` when no candidate survives. @@ -169,7 +176,7 @@ where Some(ServingConfig { local_site: overlay.local_site.clone(), window_secs: WINDOW_SECS, - load_window_ms: LOAD_WINDOW_MS, + load_window_ms: load_window_ms(inputs.scrape_interval), candidates, peers, }) @@ -263,12 +270,41 @@ fn candidates(overlay: &RoutingOverlay) -> Vec { /// An inference candidate the gateway will accept, whatever its admission. /// /// A candidate not taking new requests stays in, so the gateway can tell a known -/// down model, answered with 503, from an unknown one. +/// down model, answered with 503, from an unknown one. Its site must be a DNS-1123 +/// label, as enrolled site names are: the gateway carries it in headers and ids. fn routable(candidate: &RoutingCandidate) -> bool { candidate.kind == INFERENCE_MODEL && [&candidate.name, &candidate.site, &candidate.cluster] .into_iter() .all(|id| valid_id(id)) + && certs::validate_site_name(&candidate.site).is_ok() +} + +/// The candidate sites the serving config refuses for not being DNS-1123 labels. +fn refused_sites(overlay: &RoutingOverlay) -> BTreeSet { + overlay + .candidates + .iter() + .chain(&overlay.excluded) + .filter(|candidate| certs::validate_site_name(&candidate.site).is_err()) + .map(|candidate| candidate.site.clone()) + .collect() +} + +/// The refused sites last seen per `GridNetwork`, keyed by network name. +pub(crate) type RefusedSites = Mutex>>; + +/// Warn when `network`'s set of refused sites changes, so an operator sees why a site never +/// routes. Kept per network, so two networks do not flip each other's warning. +pub(crate) fn warn_refused_sites(overlay: &RoutingOverlay, network: &str, last: &RefusedSites) { + let refused = refused_sites(overlay); + let mut last = last.lock().unwrap_or_else(PoisonError::into_inner); + if last.get(network) != Some(&refused) { + if !refused.is_empty() { + tracing::warn!(network, sites = ?refused, "serving config: refusing candidates whose site is not a DNS-1123 label"); + } + last.insert(network.to_owned(), refused); + } } /// Non-blank and within the gateway's identifier bound. @@ -422,6 +458,7 @@ mod tests { tls_mount: "/etc/praxis/tls", local_signals_addr: None, pins: &NO_PINS, + scrape_interval: Duration::from_secs(5), }; fn cand(name: &str, site: &str, cluster: &str, admission: Option<&str>) -> RoutingCandidate { @@ -460,6 +497,21 @@ mod tests { .collect() } + #[test] + fn the_load_window_covers_the_oldest_a_sample_can_arrive_plus_a_poll() { + assert_eq!( + load_window_ms(Duration::from_secs(5)), + 17_000, + "5s poll + 5s scrape + 2s timeout + 5s" + ); + assert_eq!(load_window_ms(Duration::from_secs(1)), 13_000); + let oldest_arrival = PEER_INTERVAL_MS + 5_000 + PEER_TIMEOUT_MS; + assert!( + i64::try_from(oldest_arrival).unwrap_or(i64::MAX) < load_window_ms(Duration::from_secs(5)), + "a sample at the worst-case age is still fresh" + ); + } + #[test] fn renders_the_contract_the_gateway_parses() { let members = [("site-b", "203.0.113.7:9091"), ("site-a", "198.51.100.1:9091")]; @@ -489,6 +541,31 @@ mod tests { assert_eq!(order, ["site-a", "site-b"], "local first, one entry per tuple"); } + #[test] + fn refused_sites_name_only_the_invalid_ones() { + let sites = refused_sites(&overlay(vec![ + cand("llama", "site-a", "pool-a", None), + cand("llama", "site.b", "pool-b", None), + ])); + assert_eq!(sites, BTreeSet::from(["site.b".to_owned()])); + } + + #[test] + fn refused_sites_are_kept_per_network() { + let last = RefusedSites::default(); + let bad = overlay(vec![cand("llama", "site.b", "pool-b", None)]); + let good = overlay(vec![cand("llama", "site-a", "pool-a", None)]); + warn_refused_sites(&bad, "east", &last); + warn_refused_sites(&good, "west", &last); + let held = last.into_inner().unwrap_or_else(PoisonError::into_inner); + assert_eq!(held.get("east"), Some(&BTreeSet::from(["site.b".to_owned()]))); + assert_eq!( + held.get("west"), + Some(&BTreeSet::new()), + "another network does not clear it" + ); + } + #[test] fn candidates_the_gateway_would_reject_are_dropped() { let cases = [ @@ -497,6 +574,10 @@ mod tests { "oversized name", cand(&"m".repeat(MAX_NAME_LEN + 1), "site-b", "pool-b", None), ), + ("site with a dot", cand("llama", "site.b", "pool-b", None)), + ("site with a slash", cand("llama", "site/b", "pool-b", None)), + ("site with a quote", cand("llama", "site\"b", "pool-b", None)), + ("uppercase site", cand("llama", "Site-B", "pool-b", None)), ]; for (label, bad) in cases { assert!(render(&overlay(vec![bad]), [], &INPUTS).is_none(), "{label}"); @@ -594,10 +675,9 @@ mod tests { let other = render(&two_site(), [("site-z", "203.0.113.9:9091")], &INPUTS).expect("routable"); assert!(other.peers.is_empty(), "a site with no candidate is not polled"); let bad_name = overlay(vec![cand("llama", "Site_B", "pool-b", None)]); - let config = render(&bad_name, [("Site_B", "203.0.113.7:9091")], &INPUTS).expect("routable"); assert!( - config.peers.is_empty(), - "a non-DNS site name would fail the gateway's SNI" + render(&bad_name, [("Site_B", "203.0.113.7:9091")], &INPUTS).is_none(), + "a non-DNS site name is neither a candidate nor a peer" ); } From e422e0cd47dfee07b6048009764a6402d5bc7539 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 19:07:36 -0400 Subject: [PATCH 08/10] fix(operator): keep a busy pool that still answers out of NoEndpointsReady The EPP's ready-endpoint count is endpoints with fresh metrics, so a saturated engine whose /metrics answers late reads as zero while it serves, and excluding it herds its load onto the other sites. Zero counts now exclude only when the EPP also recorded no first token, completion or usage report for the pool in the last 30s. The running averages are no evidence: they freeze when the count is zero. Signed-off-by: Sam Batschelet --- docs/architecture/crds.md | 2 +- docs/architecture/polling-metrics.md | 4 +- operator/src/latency.rs | 12 +++ operator/src/readiness.rs | 155 ++++++++++++++++++++++++--- 4 files changed, 159 insertions(+), 14 deletions(-) diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 81e41f98..160806f4 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -571,7 +571,7 @@ says nothing about whether it serves now. | Status | Reason | When | |---|---|---| | `True` | `Ready` | The latest scrape succeeded with at least one ready endpoint. | -| `False` | `NoEndpointsReady` | Two consecutive scrapes counted zero ready endpoints. | +| `False` | `NoEndpointsReady` | Two consecutive scrapes counted zero ready endpoints, and the EPP recorded no engine answer in the last 30s. | | `False` | `NoLivenessCheck` | The scrape answered without the pool's ready-endpoint series (`llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`, for `poolName`). The message names what was missing. | | `False` | `ScrapeTimedOut` | No scrape succeeded within `staleMetricsSeconds`, and the latest timed out. | | `False` | `ScrapeUnauthorized` | As above, and the latest was refused with 401 or 403. | diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index 385a7550..e7cc3d0d 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -56,7 +56,9 @@ request path does not read raw signals or compute load. It reads resolved order. Each site's operator decides whether each of its providers can serve now and publishes the verdict as `grid_provider_ready{grid_site,grid_provider}`: 1 when ready, 0 when not. A provider is not ready when its EPP reports zero ready -endpoints for two scrapes in a row, when a scrape answers without the pool's +endpoints for two scrapes in a row and recorded no engine answer in the last 30s +(the EPP counts endpoints with fresh metrics, so a saturated engine can read as +zero while it serves), when a scrape answers without the pool's ready-endpoint series, when no scrape has succeeded within `staleMetricsSeconds` (half the signal TTL when unset), or when the provider is `Unavailable`. The same verdict is the provider's `Ready` condition, whose reason names the cause and, for a diff --git a/operator/src/latency.rs b/operator/src/latency.rs index 18c05a47..6afaaf97 100644 --- a/operator/src/latency.rs +++ b/operator/src/latency.rs @@ -140,6 +140,13 @@ impl Snapshot { } } + /// Whether the engine answered since `earlier`: a first token, a completion, or a usage report. + pub(crate) fn produced_since(&self, earlier: &Self) -> bool { + self.ttft.count > earlier.ttft.count + || self.tpot.count > earlier.tpot.count + || self.input.count > earlier.input.count + } + /// `self - earlier`, `None` when any counter went backwards. fn since(&self, earlier: &Self) -> Option { let optional = |now: Option, then: Option| match (now, then) { @@ -222,6 +229,11 @@ pub(crate) struct History { } impl History { + /// The latest recorded scrape. + pub(crate) fn latest(&self) -> Option<&Snapshot> { + self.snapshots.back().map(|(_, snapshot)| snapshot) + } + /// Record this scrape and return the latency series to publish for it. pub(crate) fn record(&mut self, snapshot: Snapshot, now: Instant) -> Vec { while self diff --git a/operator/src/readiness.rs b/operator/src/readiness.rs index 7bea43ff..a717c49c 100644 --- a/operator/src/readiness.rs +++ b/operator/src/readiness.rs @@ -22,6 +22,12 @@ pub(crate) const DEFAULT_READY_ENDPOINTS: [&str; 2] = ["llm_d_epp_ready_endpoint /// pool that just drained can read ready once before its first request lands. pub(crate) const STREAK: u32 = 2; +/// How recently the engine must have answered for a zero count to read as busy, not down. +/// +/// The EPP counts endpoints whose metrics are fresh, so a saturated engine whose /metrics +/// answers late reads as zero while it still serves. +pub(crate) const PROGRESS_WINDOW: Duration = Duration::from_secs(30); + /// The series this site publishes per provider declaring `maxRunning`: that capacity. pub const CAPACITY_SIGNAL: &str = "grid_provider_capacity_requests"; @@ -213,6 +219,10 @@ struct Probe { ready_streak: u32, /// Whether the count marks the provider not ready, with [`STREAK`] hysteresis both ways. no_endpoints: bool, + /// When the EPP last recorded an engine answer for this provider. + last_progress: Option, + /// Whether the latest zero count came while the engine was answering. + busy: bool, /// Recent EPP latency snapshots. latency: crate::latency::History, } @@ -242,6 +252,7 @@ impl ReadinessStore { } /// Record a scrape's latency counters and return the latency series it publishes. + #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] pub(crate) fn record_latency( &self, key: &str, @@ -251,11 +262,19 @@ impl ReadinessStore { ) -> Vec { // Built before locking, so the store is held only to append it. let snapshot = crate::latency::Snapshot::of(observations, pool); - self.probes() - .entry(key.to_owned()) - .or_default() - .latency - .record(snapshot, now) + // The counters carry no pool label, so an EPP serving several pools proves nothing. + let attributable = !serves_several_pools(observations); + let mut probes = self.probes(); + let probe = probes.entry(key.to_owned()).or_default(); + if attributable + && probe + .latency + .latest() + .is_some_and(|earlier| snapshot.produced_since(earlier)) + { + probe.last_progress = Some(now); + } + probe.latency.record(snapshot, now) } /// Record a successful scrape at `now`, with `missing` naming the series it lacked. @@ -276,17 +295,22 @@ impl ReadinessStore { Some(count) if count < 1.0 => { probe.zero_streak = probe.zero_streak.saturating_add(1); probe.ready_streak = 0; - probe.no_endpoints |= probe.zero_streak >= STREAK; + probe.busy = probe + .last_progress + .is_some_and(|at| now.saturating_duration_since(at) <= PROGRESS_WINDOW); + probe.no_endpoints |= probe.zero_streak >= STREAK && !probe.busy; }, Some(_) => { probe.ready_streak = probe.ready_streak.saturating_add(1); probe.zero_streak = 0; + probe.busy = false; probe.no_endpoints &= probe.ready_streak < STREAK; }, // No count to read: reachable is all that is known. None => { probe.zero_streak = 0; probe.ready_streak = 0; + probe.busy = false; probe.no_endpoints = false; }, } @@ -407,14 +431,23 @@ fn from_ready_endpoints(probe: &Probe) -> Verdict { .unwrap_or_else(|| "metrics reachable, but no ready-endpoint series".to_owned()), }; }; - let message = if count < 1.0 { - "0 ready endpoints in the latest scrape only".to_owned() - } else { - format!("{count} ready endpoints") - }; Verdict { reason: Reason::Ready, - message, + message: ready_message(count, probe.busy), + } +} + +/// The detail for a ready provider counting `count` endpoints. +fn ready_message(count: f64, busy: bool) -> String { + if count >= 1.0 { + format!("{count} ready endpoints") + } else if busy { + format!( + "0 ready endpoints, but the engine answered in the last {}s", + PROGRESS_WINDOW.as_secs() + ) + } else { + "0 ready endpoints in the latest scrape only".to_owned() } } @@ -675,6 +708,104 @@ mod tests { assert_eq!(scrape(1.0), Some(Reason::Ready)); } + /// One EPP scrape for pool qwen3: its ready count, a running average frozen at 128, and + /// `answered` usage reports so far. + fn saturated(ready: f64, answered: f64) -> Vec { + let unlabeled = |metric: &str, value: f64| Observation { + metric: metric.to_owned(), + labels: BTreeMap::new(), + value, + timestamp_ms: None, + }; + vec![ + sample("llm_d_epp_ready_endpoints", "qwen3", ready), + sample("llm_d_epp_average_running_requests", "qwen3", 128.0), + unlabeled("llm_d_epp_request_input_tokens_count", answered), + unlabeled("llm_d_epp_request_input_tokens_sum", answered * 100.0), + ] + } + + /// Record `observations` as the signals loop does, and return the verdict. + fn scrape_at(store: &ReadinessStore, observations: &[Observation], now: Instant) -> Option { + let ready = ready_endpoints(observations, &DEFAULT_READY_ENDPOINTS, Some("qwen3")); + store.record_latency("p", observations, Some("qwen3"), now); + store.record_success("p", ready, None, observations.to_vec(), now); + reason(store, now) + } + + const STEP: Duration = Duration::from_secs(5); + + #[test] + fn a_saturated_pool_that_still_answers_stays_ready_on_zero_counts() { + let store = ReadinessStore::default(); + let start = Instant::now(); + assert_eq!(scrape_at(&store, &saturated(2.0, 100.0), start), Some(Reason::Ready)); + for (n, answered) in [(1_u32, 140.0), (2, 180.0), (3, 220.0)] { + assert_eq!( + scrape_at(&store, &saturated(0.0, answered), start + STEP * n), + Some(Reason::Ready), + "scrape {n}: running 128 and answering is busy, not down" + ); + } + let verdict = store.verdict("p", false, STALE, start + STEP * 3).unwrap(); + assert!(verdict.message.contains("answered"), "{}", verdict.message); + } + + #[test] + fn a_zero_count_with_no_answers_excludes_even_with_running_requests() { + let store = ReadinessStore::default(); + let start = Instant::now(); + assert_eq!(scrape_at(&store, &saturated(2.0, 100.0), start), Some(Reason::Ready)); + assert_eq!( + scrape_at(&store, &saturated(0.0, 100.0), start + STEP), + Some(Reason::Ready) + ); + assert_eq!( + scrape_at(&store, &saturated(0.0, 100.0), start + STEP * 2), + Some(Reason::NoEndpointsReady), + "the running average freezes at zero endpoints, so it is no evidence" + ); + } + + #[test] + fn a_pool_that_stops_answering_is_excluded_once_the_window_passes() { + let store = ReadinessStore::default(); + let start = Instant::now(); + scrape_at(&store, &saturated(2.0, 100.0), start); + let answered = start + STEP; + scrape_at(&store, &saturated(0.0, 140.0), answered); + // Six more scrapes reach 30s after the last answer: still inside the window. + for n in 1..=6_u32 { + assert_eq!( + scrape_at(&store, &saturated(0.0, 140.0), answered + STEP * n), + Some(Reason::Ready), + "{}s after the last answer", + (STEP * n).as_secs() + ); + } + assert_eq!( + scrape_at(&store, &saturated(0.0, 140.0), answered + STEP * 7), + Some(Reason::NoEndpointsReady) + ); + } + + #[test] + fn answers_from_an_epp_serving_several_pools_prove_nothing() { + let store = ReadinessStore::default(); + let start = Instant::now(); + let with_other_pool = |ready, answered| { + let mut observations = saturated(ready, answered); + observations.push(sample("llm_d_epp_ready_endpoints", "llama", 3.0)); + observations + }; + scrape_at(&store, &with_other_pool(2.0, 100.0), start); + scrape_at(&store, &with_other_pool(0.0, 140.0), start + STEP); + assert_eq!( + scrape_at(&store, &with_other_pool(0.0, 180.0), start + STEP * 2), + Some(Reason::NoEndpointsReady) + ); + } + #[test] fn a_first_scrape_that_fails_gets_the_staleness_window_before_a_verdict() { let store = ReadinessStore::default(); From ad32e5e2151cbc4945de40c9c1474891672218db Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 19:20:06 -0400 Subject: [PATCH 09/10] fix(operator): read latency from the EPP's histograms and counters The provider scrape went through the peer-relay parser, which drops counters and histogram parts, so no latency, error ratio or engine-answer evidence was ever computed. A local scrape now keeps every sample for this site's windows and republishes gauges only, as before. Signed-off-by: Sam Batschelet --- operator/src/latency.rs | 6 +- operator/src/resources/provider_metrics.rs | 106 ++++++++++++++++++++- operator/src/signals.rs | 21 +++- 3 files changed, 126 insertions(+), 7 deletions(-) diff --git a/operator/src/latency.rs b/operator/src/latency.rs index 6afaaf97..de637667 100644 --- a/operator/src/latency.rs +++ b/operator/src/latency.rs @@ -210,8 +210,10 @@ impl Snapshot { /// Mean TTFT less mean flow-control wait, over mean uncached input tokens. /// - /// Sensitive to the workload mix: TTFT counts streaming requests only, while token - /// counts cover every request. + /// Token means are per usage report. An engine reporting usage on every chunk weights + /// long outputs more, but each report carries the whole prompt, so the mean stays a + /// prompt length. Sensitive to the workload mix: TTFT counts streaming requests only, + /// while token counts cover every request. fn prefill_seconds_per_token(&self) -> Option { let wait = self.queue_wait.and_then(SumCount::mean).unwrap_or(0.0); let cached = self.cached.map_or(0.0, |cached| cached.sum / self.input.count); diff --git a/operator/src/resources/provider_metrics.rs b/operator/src/resources/provider_metrics.rs index 0bd4eb59..9a064149 100644 --- a/operator/src/resources/provider_metrics.rs +++ b/operator/src/resources/provider_metrics.rs @@ -147,7 +147,8 @@ fn record_scrape( plan: &SignalScrapePlan<'_>, text: &str, ) { - let parsed = crate::signals::parse(text); + // Every sample feeds this site's own windows; only gauges are republished. + let (parsed, republishable): (Vec<_>, Vec<_>) = crate::signals::parse_scrape(text).into_iter().unzip(); let pool = provider .spec .metrics_config @@ -159,6 +160,8 @@ fn record_scrape( let latency = readiness.record_latency(key, &parsed, pool, Instant::now()); let observations = parsed .into_iter() + .zip(republishable) + .filter_map(|(o, republishable)| republishable.then_some(o)) .filter(|o| plan.wanted.contains(o.metric.as_str())) // A local sample's freshness is its collection time, so drop any trailing // timestamp. Only relayed peer samples carry a per-sample stamp. @@ -845,6 +848,107 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + /// A `KServe` EPP exposition for pool `qwen3-kserve`, in the label sets the lab EPP emits: + /// `served` streaming requests answered, about 104 usage reports each. + #[expect(clippy::too_many_lines, reason = "one exposition, line for line")] + fn kserve_epp_exposition(served: f64, ready: f64) -> String { + let l = r#"fairness_id="default-flow",model_name="qwen3",priority="0",target_model_name="qwen3""#; + let (b01, b05, b1) = (served * 0.2, served * 0.6, served * 0.9); + let (ttft_sum, tpot_sum) = (served * 0.4, served * 0.02); + let (reports, tokens) = (served * 104.0, served * 104.0 * 300.0); + let (requests, errors) = (served + 14.0, served / 100.0); + format!( + r#"# HELP llm_d_epp_ready_endpoints Ready endpoints. +# TYPE llm_d_epp_ready_endpoints gauge +llm_d_epp_ready_endpoints{{name="qwen3-kserve"}} {ready} +# TYPE llm_d_epp_average_running_requests gauge +llm_d_epp_average_running_requests{{name="qwen3-kserve"}} 128 +# TYPE llm_d_epp_request_ttft_seconds histogram +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="false",le="0.1"}} 14 +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="false",le="+Inf"}} 14 +llm_d_epp_request_ttft_seconds_sum{{{l},streaming="false"}} 1.9 +llm_d_epp_request_ttft_seconds_count{{{l},streaming="false"}} 14 +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="0.1"}} {b01} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="0.5"}} {b05} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="1"}} {b1} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="2.5"}} {served} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="+Inf"}} {served} +llm_d_epp_request_ttft_seconds_sum{{{l},streaming="true"}} {ttft_sum} +llm_d_epp_request_ttft_seconds_count{{{l},streaming="true"}} {served} +# TYPE llm_d_epp_request_streaming_tpot_seconds histogram +llm_d_epp_request_streaming_tpot_seconds_sum{{{l}}} {tpot_sum} +llm_d_epp_request_streaming_tpot_seconds_count{{{l}}} {served} +# TYPE llm_d_epp_request_input_tokens histogram +llm_d_epp_request_input_tokens_sum{{{l}}} {tokens} +llm_d_epp_request_input_tokens_count{{{l}}} {reports} +# TYPE llm_d_epp_request_total counter +llm_d_epp_request_total{{{l}}} {requests} +# TYPE llm_d_epp_request_error_total counter +llm_d_epp_request_error_total{{{l},error_code="503"}} {errors} +"# + ) + } + + #[test] + #[expect(clippy::too_many_lines, reason = "two scrapes and every published series")] + fn a_kserve_epp_scrape_publishes_latency_and_republishes_gauges_only() { + let readiness = crate::readiness::ReadinessStore::default(); + let mut mc = mc_with_queue("llm_d_epp_average_running_requests"); + mc.pool_name = Some("qwen3-kserve".to_owned()); + // A counter declared as a signal is still never republished. + mc.signal_names.error_rate = Some("llm_d_epp_request_total".to_owned()); + let provider = provider_fixture("qwen3-hq-east", "http://epp:9090", Some(mc)); + let plan = signal_scrape_plan(&provider).unwrap_or_else(|| std::process::abort()); + record_scrape( + &readiness, + "net/p", + &provider, + &plan, + &kserve_epp_exposition(11_000.0, 2.0), + ); + assert!(readiness.take_fresh("net/p").is_some(), "first scrape recorded"); + record_scrape( + &readiness, + "net/p", + &provider, + &plan, + &kserve_epp_exposition(11_581.0, 0.0), + ); + let published = readiness.take_fresh("net/p").unwrap_or_default(); + let value = |name: &str| published.iter().find(|o| o.metric == name).map(|o| o.value); + for name in [ + crate::latency::TTFT_P50_SIGNAL, + crate::latency::TTFT_P90_SIGNAL, + crate::latency::TPOT_SIGNAL, + crate::latency::PREFILL_SIGNAL, + crate::latency::ERROR_RATIO_SIGNAL, + ] { + assert!(value(name).is_some(), "{name} missing from {published:?}"); + } + assert!((value(crate::latency::TPOT_SIGNAL).unwrap_or_default() - 0.02).abs() < 1e-9); + // 581 answers at 0.4s mean TTFT over 300 prompt tokens per usage report. + let prefill = value(crate::latency::PREFILL_SIGNAL).unwrap_or_default(); + assert!((prefill - 0.4 / 300.0).abs() < 1e-9, "prefill {prefill}"); + assert!( + published.iter().all(|o| !o.metric.ends_with("_total") + && !o.metric.ends_with("_count") + && !o.metric.ends_with("_sum") + && !o.metric.ends_with("_bucket")), + "counters and histogram parts stay local: {published:?}" + ); + assert!( + value("llm_d_epp_average_running_requests").is_some(), + "declared gauge republished" + ); + let verdict = readiness + .verdict("net/p", false, Duration::from_secs(15), Instant::now()) + .unwrap_or_else(|| std::process::abort()); + assert!( + verdict.message.contains("answered"), + "progress seen through the parser: {verdict:?}" + ); + } + fn mc_with_queue(metric_name: &str) -> MetricsConfig { MetricsConfig { path: "/metrics".to_owned(), diff --git a/operator/src/signals.rs b/operator/src/signals.rs index fa9110ae..406bbae9 100644 --- a/operator/src/signals.rs +++ b/operator/src/signals.rs @@ -67,6 +67,18 @@ pub struct Observation { /// somebody else's scrape and the caller cannot repair it. #[must_use] pub fn parse(text: &str) -> Vec { + parse_scrape(text) + .into_iter() + .filter_map(|(o, republishable)| republishable.then_some(o)) + .collect() +} + +/// Parse a scrape this site made itself: every finite sample, each paired with whether +/// [`parse`] would keep it for republishing. +/// +/// Counters and histogram parts stay, for this site's own latency and error windows. +#[must_use] +pub(crate) fn parse_scrape(text: &str) -> Vec<(Observation, bool)> { // Types first: a declaration may follow its samples, and reading in one // pass would admit a counter that had not been typed yet. let types: HashMap<&str, &str> = text @@ -81,13 +93,14 @@ pub fn parse(text: &str) -> Vec { text.lines() .filter(|l| !l.trim_start().starts_with('#')) .filter_map(parse_sample) - .filter(|o| { + .filter(|o| o.value.is_finite()) + .map(|o| { // Gauges and untyped only: a relayed counter reports our restarts, // and an aggregate we did not observe cannot be recombined. The name // check catches aggregates that arrive untyped. - matches!(types.get(o.metric.as_str()), None | Some(&("gauge" | "untyped"))) - && !is_aggregate_part(&o.metric) - && o.value.is_finite() + let republishable = matches!(types.get(o.metric.as_str()), None | Some(&("gauge" | "untyped"))) + && !is_aggregate_part(&o.metric); + (o, republishable) }) .collect() } From 1c59e6e7839941966ef351d9f31b68c677fbb1a8 Mon Sep 17 00:00:00 2001 From: Sam Batschelet Date: Sun, 4 Oct 2026 19:57:19 -0400 Subject: [PATCH 10/10] feat(gateway): choose sites by published saturation per capacity Each site's load is rho, the grid_provider_saturation_ratio its operator publishes, read when the snapshot is built. Among healthy sites with room (rho below 1): two choices drawn by capacity at three or more, taking the lower rho; a draw weighted by capacity over 1 + rho at two, each share kept within 0.1 to 0.9; or the only one. With none, a capacity draw among the healthy sites tied on the best polled queue depth, which also covers sites that publish no load. Capacity is unknown, not 1, when unpublished. Selection no longer takes the front match of a tied list, which pinned every request to the first local cluster whenever sites tied, at zero load or at the same queue depth. A candidate whose latest grid_provider_ready sample is 0 is excluded, and one on a cluster praxis reports with no healthy endpoint is ordered last. Demotion applies only once praxis has reported health, so an unobserved registry demotes nothing. A model sheds when every healthy site serving it holds rho of at least 1.05, and routes again once one reaches 0.95, decided per snapshot. A shed answers 429 with Retry-After and an OpenAI-style error, type rate_limit_exceeded and code capacity_exhausted, so a client backs off as it does for any overload. No healthy site answers 503 no_healthy_site, an outage rather than load, and an unknown model 404 model_not_found. grid_route_decisions_total counts every outcome by site and a closed reason set, served with each site's score on an opt-in TLS metrics listener that the chart exposes with a Service and ServiceMonitor. A Narrow stage can clear sites with room before the draw, never add one. It sees each site's rho and the recent latency its operator publishes, which ranking does not use. Draws mix a per-request turn that starts at a random seed, so replicas do not draw in step. One request weighs at most 64 sites with room. docs/site-selection.md is a tuning guide for this path. Signed-off-by: Sam Batschelet --- charts/praxis-gateway/README.md | 23 + charts/praxis-gateway/templates/_helpers.tpl | 33 + .../praxis-gateway/templates/deployment.yaml | 28 +- .../templates/networkpolicy.yaml | 11 + .../templates/service-metrics.yaml | 24 + .../templates/servicemonitor.yaml | 35 + .../tests/metrics_listener_test.yaml | 159 +++ charts/praxis-gateway/values.schema.json | 30 + charts/praxis-gateway/values.yaml | 28 + docs/README.md | 2 + docs/architecture/polling-metrics.md | 27 + docs/routing.md | 2 + docs/site-selection.md | 132 +++ gateway/Cargo.lock | 8 + gateway/Cargo.toml | 7 + gateway/ai-grid-filters/Cargo.toml | 6 +- gateway/ai-grid-filters/src/control.rs | 422 ++++---- gateway/ai-grid-filters/src/decisions.rs | 133 +++ gateway/ai-grid-filters/src/descriptor.rs | 63 +- gateway/ai-grid-filters/src/flow.rs | 60 +- gateway/ai-grid-filters/src/health.rs | 138 +++ gateway/ai-grid-filters/src/lib.rs | 11 +- gateway/ai-grid-filters/src/route.rs | 976 +++++++++++++++++- gateway/ai-grid-filters/src/serving.rs | 60 +- gateway/ai-grid-filters/src/snapshot.rs | 508 ++++++++- gateway/src/main.rs | 30 +- gateway/src/metrics_listener.rs | 471 +++++++++ gateway/tests/chart_renders.rs | 116 +++ gateway/tests/metrics_listener.rs | 198 ++++ gateway/tests/provider_readiness.rs | 512 +++++++++ gateway/tests/unrouted.rs | 218 ++++ signals/src/signals.rs | 19 +- ...luster-multi-gateway-3286232.resolved.yaml | 530 ++++++++++ 33 files changed, 4754 insertions(+), 266 deletions(-) create mode 100644 charts/praxis-gateway/templates/service-metrics.yaml create mode 100644 charts/praxis-gateway/templates/servicemonitor.yaml create mode 100644 charts/praxis-gateway/tests/metrics_listener_test.yaml create mode 100644 docs/site-selection.md create mode 100644 gateway/ai-grid-filters/src/decisions.rs create mode 100644 gateway/ai-grid-filters/src/health.rs create mode 100644 gateway/src/metrics_listener.rs create mode 100644 gateway/tests/chart_renders.rs create mode 100644 gateway/tests/metrics_listener.rs create mode 100644 gateway/tests/provider_readiness.rs create mode 100644 gateway/tests/unrouted.rs create mode 100644 tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index f86c8f82..163a4843 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -228,6 +228,10 @@ Praxis AI image; these values may advance independently. | `gatewayConfig.auth.validateCA` | object | empty | CA for the validate call (`configMap` or `secret`, `key`). Set as `SSL_CERT_FILE`, which replaces the platform trust store for the validate call and https backends without a per-backend CA or `upstreamCA`. mutual_tls backends and `upstreamCA` are unaffected. See the recipe below. | | `networkPolicy.enabled` | bool | `false` | Render a NetworkPolicy that limits which pods can reach the listener port, where the CNI enforces NetworkPolicy. It is not authentication. Node and host-network traffic handling is CNI-specific (OVN-Kubernetes: the `policy-group.network.openshift.io/host-network` label), and a LoadBalancer with `externalTrafficPolicy: Cluster` can SNAT clients to node IPs. | | `networkPolicy.from` | list | `[]` | NetworkPolicyPeer entries allowed in. Required when enabled. With `auth.mode: none`, list only the authenticating front. `{podSelector: {}}` admits every pod in this namespace. An empty `namespaceSelector` and an `ipBlock` of `0.0.0.0/0` or `::/0` admit everyone and fail the render. An all-address `ipBlock` with `except` entries is allowed. The check reads selector emptiness and the cidr only, so `matchExpressions` that happen to select every pod pass. A provider gateway behind a LoadBalancer that SNATs clients to node IPs needs `ipBlock` peers for those node addresses. | +| `metricsListener.enabled` | bool | `false` | Serve `GET /metrics` over TLS on its own port and ClusterIP Service, for an in-cluster Prometheus. The admin listener refuses a non-loopback Host, so Prometheus cannot scrape it. Needs the grid-gateway image, `existingSecret`, `fromNamespaces`, and `networkPolicy.enabled`. The port answers only `/metrics` but has no authentication, so the NetworkPolicy is its access control, and that holds only where the CNI enforces NetworkPolicy. | +| `metricsListener.existingSecret` | string | `""` | Secret with `tls.crt` and `tls.key`. On OpenShift, request it with `metricsListener.service.annotations` `service.beta.openshift.io/serving-cert-secret-name`. The listener reloads the cert when the Secret changes. | +| `metricsListener.fromNamespaces` | list | `[]` | Namespace names allowed to reach the metrics port, for example `openshift-user-workload-monitoring`. | +| `metricsListener.serviceMonitor.enabled` | bool | `false` | Render a ServiceMonitor that verifies the cert against `caConfigMap` (for example `openshift-service-ca.crt`, key `service-ca.crt`) and renames Praxis's `cluster` label to `backend`, since ACM uses `cluster` for the managed cluster. | | `gatewayConfig.upstreamCA.secretName` | string | `""` | CA bundle for backend TLS without a per-cluster CA (`upstream_ca_file`). | | `gatewayConfig.listenerTls.enabled` | bool | `false` | Terminate TLS at the listener from `existingSecret`, in render or BYO mode. Names the port `https`. The cert mounts at `listenerTls.mountPath` (`/etc/praxis/listener-tls`), so a BYO config moving off `tls.enabled` must point its listener `cert_path`/`key_path` there. On OpenShift, annotate the Service with `service.beta.openshift.io/serving-cert-secret-name`. | | `port.containerPort` | int | `8080` | Container port. | @@ -274,6 +278,25 @@ Praxis AI image; these values may advance independently. | `topologySpreadConstraints` | list | `[]` | Topology spread constraints. | | `priorityClassName` | string | `""` | Pod priority class. | +## Metrics + +The metrics listener serves the Praxis registry, including the grid gateway's own series. With the ServiceMonitor, Praxis's `cluster` label arrives as `backend`. + +| Metric | Labels | Meaning | +|--------|--------|---------| +| `grid_route_decisions_total` | `site`, `reason` | Requests `grid_site_route` decided. `site` is a site name from the serving config, or empty for a refusal. No label comes from the request. | +| `grid_route_site_score` | `site`, `cluster` | The queue depth the last route order used for each candidate, lower first. `inf` when unmeasured, `NaN` when excluded or demoted, or when the pair left the topology. | + +`reason` is one of five values, and adding one is a deliberate change: + +| `reason` | `site` | Response | +|----------|--------|----------| +| `routed` | chosen site | Sent to a healthy site. How it ranked is in `grid_route_site_score`. | +| `fallback` | chosen site | Sent to a demoted site because no healthy one was left. | +| `not_ready` | empty | 503: every candidate was excluded. | +| `no_route` | empty | 503: an admitted candidate had no route from this gateway. | +| `bad_request` | empty | 400 or 404: no model, or a model no candidate serves. | + ## Security The chart enforces Kubernetes restricted security defaults: diff --git a/charts/praxis-gateway/templates/_helpers.tpl b/charts/praxis-gateway/templates/_helpers.tpl index b09ec3ba..5de33a5f 100644 --- a/charts/praxis-gateway/templates/_helpers.tpl +++ b/charts/praxis-gateway/templates/_helpers.tpl @@ -555,3 +555,36 @@ RUST_LOG for the gateway and overlay-sync: log.filter when set, else log.level, {{- $log := .Values.log | default dict -}} {{- $log.filter | default $log.level -}} {{- end }} + +{{/* +The metrics listener needs the grid-gateway image, a cert, and the NetworkPolicy that +limits its port; without the policy any pod could scrape it. +*/}} +{{- define "praxis-gateway.validateMetricsListener" -}} +{{- $m := .Values.metricsListener }} +{{- if $m.enabled }} +{{- if ne .Values.image.flavor "grid-gateway" }} +{{- fail "metricsListener needs image.flavor grid-gateway" }} +{{- end }} +{{- if not $m.existingSecret }} +{{- fail "metricsListener.existingSecret is required: the listener serves TLS only" }} +{{- end }} +{{- if not .Values.networkPolicy.enabled }} +{{- fail "metricsListener needs networkPolicy.enabled, which limits the metrics port to metricsListener.fromNamespaces" }} +{{- end }} +{{- if not $m.fromNamespaces }} +{{- fail "metricsListener.fromNamespaces needs at least one namespace" }} +{{- end }} +{{- $taken := list (int .Values.port.containerPort) }} +{{- if and .Values.overlay.enabled .Values.overlay.sidecar.enabled }}{{ $taken = append $taken 9091 }}{{ end }} +{{- if has (int $m.port) $taken }} +{{- fail (printf "metricsListener.port %d collides with another gateway pod port" (int $m.port)) }} +{{- end }} +{{- end }} +{{- if and $m.serviceMonitor.enabled (not $m.enabled) }} +{{- fail "metricsListener.serviceMonitor needs metricsListener.enabled" }} +{{- end }} +{{- if and $m.serviceMonitor.enabled (not $m.serviceMonitor.caConfigMap.name) }} +{{- fail "metricsListener.serviceMonitor.caConfigMap.name is required to verify the metrics cert" }} +{{- end }} +{{- end }} diff --git a/charts/praxis-gateway/templates/deployment.yaml b/charts/praxis-gateway/templates/deployment.yaml index 4952793e..db0790b3 100644 --- a/charts/praxis-gateway/templates/deployment.yaml +++ b/charts/praxis-gateway/templates/deployment.yaml @@ -3,6 +3,7 @@ {{- include "praxis-gateway.validateConfig" . }} {{- include "praxis-gateway.validateMounts" . }} {{- include "praxis-gateway.validateProbes" . }} +{{- include "praxis-gateway.validateMetricsListener" . }} apiVersion: apps/v1 kind: Deployment metadata: @@ -192,7 +193,7 @@ spec: {{- $userLog := false }} {{- range .Values.env }}{{- if eq .name "RUST_LOG" }}{{- $userLog = true }}{{- end }}{{- end }} {{- $rustLog := and (not $userLog) (include "praxis-gateway.rustLog" .) }} - {{- if or $rustLog .Values.env (.Values.gridServing).enabled $withCA }} + {{- if or $rustLog .Values.env (.Values.gridServing).enabled $withCA (.Values.metricsListener).enabled }} env: {{- if $rustLog }} - name: RUST_LOG @@ -209,6 +210,16 @@ spec: - name: SSL_CERT_FILE value: {{ printf "%s/%s" $validateCA.mountPath $validateCA.key | quote }} {{- end }} + {{- with .Values.metricsListener }} + {{- if .enabled }} + - name: GRID_METRICS_ADDR + value: {{ printf "0.0.0.0:%d" (int .port) | quote }} + - name: GRID_METRICS_TLS_CERT + value: {{ printf "%s/tls.crt" .mountPath | quote }} + - name: GRID_METRICS_TLS_KEY + value: {{ printf "%s/tls.key" .mountPath | quote }} + {{- end }} + {{- end }} {{- end }} securityContext: runAsNonRoot: true @@ -227,6 +238,11 @@ spec: - name: {{ include "praxis-gateway.portName" . }} containerPort: {{ .Values.port.containerPort }} protocol: {{ .Values.port.protocol | quote }} + {{- if .Values.metricsListener.enabled }} + - name: metrics + containerPort: {{ .Values.metricsListener.port }} + protocol: TCP + {{- end }} {{- with .Values.health.readiness }} readinessProbe: {{- /* /ready fails while any backend cluster is down. */}} @@ -286,6 +302,11 @@ spec: mountPath: {{ .Values.gatewayConfig.listenerTls.mountPath | quote }} readOnly: true {{- end }} + {{- if .Values.metricsListener.enabled }} + - name: metrics-tls + mountPath: {{ .Values.metricsListener.mountPath | quote }} + readOnly: true + {{- end }} {{- if $withCA }} - name: validate-ca mountPath: {{ $validateCA.mountPath | quote }} @@ -407,6 +428,11 @@ spec: secret: secretName: {{ .Values.gatewayConfig.listenerTls.existingSecret | quote }} {{- end }} + {{- if .Values.metricsListener.enabled }} + - name: metrics-tls + secret: + secretName: {{ .Values.metricsListener.existingSecret | quote }} + {{- end }} {{- range .Values.credentials }} - name: credential-{{ .name }} secret: diff --git a/charts/praxis-gateway/templates/networkpolicy.yaml b/charts/praxis-gateway/templates/networkpolicy.yaml index 59e0b585..1e2efb32 100644 --- a/charts/praxis-gateway/templates/networkpolicy.yaml +++ b/charts/praxis-gateway/templates/networkpolicy.yaml @@ -33,4 +33,15 @@ spec: ports: - port: {{ .Values.port.containerPort }} protocol: {{ .Values.port.protocol }} + {{- if .Values.metricsListener.enabled }} + - from: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: {{ toJson .Values.metricsListener.fromNamespaces }} + ports: + - port: {{ .Values.metricsListener.port }} + protocol: TCP + {{- end }} {{- end }} diff --git a/charts/praxis-gateway/templates/service-metrics.yaml b/charts/praxis-gateway/templates/service-metrics.yaml new file mode 100644 index 00000000..f55384a7 --- /dev/null +++ b/charts/praxis-gateway/templates/service-metrics.yaml @@ -0,0 +1,24 @@ +{{- include "praxis-gateway.normalize" . }} +{{- if .Values.metricsListener.enabled }} +{{- /* ClusterIP of its own, so a LoadBalancer gateway Service never exposes metrics. */}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "praxis-gateway.fullname" . }}-metrics + labels: + {{- include "praxis-gateway.labels" . | nindent 4 }} + app.kubernetes.io/component: gateway-metrics + {{- with .Values.metricsListener.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: metrics + port: {{ .Values.metricsListener.port }} + targetPort: metrics + protocol: TCP + selector: + {{- include "praxis-gateway.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/praxis-gateway/templates/servicemonitor.yaml b/charts/praxis-gateway/templates/servicemonitor.yaml new file mode 100644 index 00000000..8355592a --- /dev/null +++ b/charts/praxis-gateway/templates/servicemonitor.yaml @@ -0,0 +1,35 @@ +{{- include "praxis-gateway.normalize" . }} +{{- with .Values.metricsListener }} +{{- if and .enabled .serviceMonitor.enabled }} +{{- $name := printf "%s-metrics" (include "praxis-gateway.fullname" $) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ $name }} + labels: + {{- include "praxis-gateway.labels" $ | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "praxis-gateway.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: gateway-metrics + endpoints: + - port: metrics + scheme: https + path: /metrics + interval: {{ .serviceMonitor.interval }} + tlsConfig: + serverName: {{ printf "%s.%s.svc" $name $.Release.Namespace }} + ca: + configMap: + name: {{ .serviceMonitor.caConfigMap.name | quote }} + key: {{ .serviceMonitor.caConfigMap.key | quote }} + metricRelabelings: + # Praxis names the upstream cluster "cluster", which ACM sets to the managed cluster. + - sourceLabels: [cluster] + regex: "(.+)" + targetLabel: backend + - action: labeldrop + regex: cluster +{{- end }} +{{- end }} diff --git a/charts/praxis-gateway/tests/metrics_listener_test.yaml b/charts/praxis-gateway/tests/metrics_listener_test.yaml new file mode 100644 index 00000000..0cbd4899 --- /dev/null +++ b/charts/praxis-gateway/tests/metrics_listener_test.yaml @@ -0,0 +1,159 @@ +suite: metrics listener +templates: + - templates/deployment.yaml + - templates/service-metrics.yaml + - templates/servicemonitor.yaml + - templates/networkpolicy.yaml + - templates/service.yaml +release: + name: gw + namespace: grid +set: + image.flavor: grid-gateway + image.repository: quay.io/sbatsche/grid-gateway + image.tag: t + networkPolicy.enabled: true + networkPolicy.from: + - podSelector: {} + metricsListener.enabled: true + metricsListener.existingSecret: grid-gateway-metrics-tls + metricsListener.fromNamespaces: [openshift-user-workload-monitoring, openshift-monitoring] +tests: + - it: renders nothing for metrics by default + set: + metricsListener.enabled: false + asserts: + - hasDocuments: + count: 0 + template: templates/service-metrics.yaml + - hasDocuments: + count: 0 + template: templates/servicemonitor.yaml + - notExists: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "GRID_METRICS_ADDR")] + template: templates/deployment.yaml + + - it: points the gateway at the listener address and the mounted cert + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "GRID_METRICS_ADDR")].value + value: "0.0.0.0:9443" + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "GRID_METRICS_TLS_KEY")].value + value: /etc/praxis/metrics-tls/tls.key + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].ports[?(@.name == "metrics")].containerPort + value: 9443 + - equal: + path: spec.template.spec.volumes[?(@.name == "metrics-tls")].secret.secretName + value: grid-gateway-metrics-tls + + - it: serves metrics on a ClusterIP Service of its own, never the gateway Service + set: + service.type: LoadBalancer + asserts: + - equal: + path: spec.type + value: ClusterIP + template: templates/service-metrics.yaml + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: gateway-metrics + template: templates/service-metrics.yaml + - notContains: + path: spec.ports + content: {name: metrics, port: 9443, targetPort: metrics, protocol: TCP} + template: templates/service.yaml + + - it: admits only the named namespaces to the metrics port + template: templates/networkpolicy.yaml + asserts: + - equal: + path: spec.ingress[1] + value: + from: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [openshift-user-workload-monitoring, openshift-monitoring] + ports: + - port: 9443 + protocol: TCP + - lengthEqual: + path: spec.ingress + count: 2 + + - it: scrapes over verified TLS and renames the praxis cluster label to backend + template: templates/servicemonitor.yaml + set: + metricsListener.serviceMonitor.enabled: true + metricsListener.serviceMonitor.caConfigMap: {name: openshift-service-ca.crt, key: service-ca.crt} + asserts: + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: gateway-metrics + - equal: + path: spec.endpoints[0].tlsConfig.serverName + value: gw-praxis-gateway-metrics.grid.svc + - equal: + path: spec.endpoints[0].tlsConfig.ca.configMap.name + value: openshift-service-ca.crt + - equal: + path: spec.endpoints[0].metricRelabelings + value: + - sourceLabels: [cluster] + regex: "(.+)" + targetLabel: backend + - action: labeldrop + regex: cluster + + - it: refuses the listener without the NetworkPolicy + template: templates/deployment.yaml + set: + networkPolicy.enabled: false + asserts: + - failedTemplate: + errorPattern: "metricsListener needs networkPolicy.enabled" + + - it: refuses the listener without a cert + template: templates/deployment.yaml + set: + metricsListener.existingSecret: "" + asserts: + - failedTemplate: + errorPattern: "metricsListener.existingSecret is required" + + - it: refuses the listener without namespaces + template: templates/deployment.yaml + set: + metricsListener.fromNamespaces: [] + asserts: + - failedTemplate: + errorPattern: "metricsListener.fromNamespaces needs at least one namespace" + + - it: refuses the listener on the ai image + template: templates/deployment.yaml + set: + image.flavor: ai + image.repository: ghcr.io/praxis-proxy/ai + asserts: + - failedTemplate: + errorPattern: "metricsListener needs image.flavor grid-gateway" + + - it: refuses a metrics port the gateway already uses + template: templates/deployment.yaml + set: + metricsListener.port: 8080 + asserts: + - failedTemplate: + errorPattern: "metricsListener.port 8080 collides" + + - it: refuses a ServiceMonitor without a CA + template: templates/deployment.yaml + set: + metricsListener.serviceMonitor.enabled: true + asserts: + - failedTemplate: + errorPattern: "caConfigMap.name is required" diff --git a/charts/praxis-gateway/values.schema.json b/charts/praxis-gateway/values.schema.json index aa48826d..dd7aaac0 100644 --- a/charts/praxis-gateway/values.schema.json +++ b/charts/praxis-gateway/values.schema.json @@ -188,6 +188,36 @@ "from": { "type": "array", "items": { "type": "object" } } } }, + "metricsListener": { + "type": "object", + "additionalProperties": false, + "description": "TLS listener serving only GET /metrics.", + "properties": { + "enabled": { "type": "boolean" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "existingSecret": { "type": "string" }, + "mountPath": { "type": "string" }, + "fromNamespaces": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { "annotations": { "type": "object" } } + }, + "serviceMonitor": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "interval": { "type": "string" }, + "caConfigMap": { + "type": "object", + "additionalProperties": false, + "properties": { "name": { "type": "string" }, "key": { "type": "string" } } + } + } + } + } + }, "replicaCount": { "type": "integer", "minimum": 1, diff --git a/charts/praxis-gateway/values.yaml b/charts/praxis-gateway/values.yaml index 57c56fc1..46934ade 100644 --- a/charts/praxis-gateway/values.yaml +++ b/charts/praxis-gateway/values.yaml @@ -292,6 +292,34 @@ networkPolicy: # - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: ingress}} # podSelector: {matchLabels: {app: authenticating-front}} +# -- TLS listener serving only GET /metrics, for an in-cluster Prometheus. The admin +# listener cannot be scraped from outside the pod. Needs the grid-gateway image and +# networkPolicy.enabled, since the port is reachable only from fromNamespaces. +metricsListener: + # -- Serve the listener. + enabled: false + # -- Container and Service port. + port: 9443 + # -- Secret holding tls.crt and tls.key, for example the serving cert that + # service.annotations requests on OpenShift. Required when enabled. + existingSecret: "" + # -- Mount path for the cert. The listener reloads it when it changes. + mountPath: "/etc/praxis/metrics-tls" + # -- Namespaces, by name, whose pods may reach the port. Required when enabled. + fromNamespaces: [] + service: + # -- Annotations on the ClusterIP metrics Service. + annotations: {} + serviceMonitor: + # -- Render a monitoring.coreos.com/v1 ServiceMonitor for the metrics Service. + enabled: false + # -- Scrape interval. + interval: 30s + # -- ConfigMap holding the CA that signed the metrics cert. Required when enabled. + caConfigMap: + name: "" + key: "ca.crt" + # -- Gateway Service configuration. service: # -- Create a Service for the gateway. diff --git a/docs/README.md b/docs/README.md index 1247e558..dc1fb74e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,8 @@ - [Routing Guide](routing.md): choose a routing behavior and configure policy, scoring, selection groups, affinity, and selection modes. +- [Tuning cross-site site selection](site-selection.md): how a grid gateway + chooses a site, what feeds it, and what to change. - [Routing Architecture and Overlay Contract](architecture/routing.md): overlay format, revision lifecycle, scope, provenance, security, and delivery. - [Provider Scoring](architecture/scoring.md): metrics input, normalization, diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index e7cc3d0d..d7d9859c 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -136,6 +136,33 @@ polls from peers, so a Prometheus that scrapes one hub sees every site. A peer's there is what the hub last polled, up to one peer poll old, not what the peer's own operator holds now. A series the operator does not hold is absent, not 0. +## Site Selection + +The gateway chooses a site by each provider's load, rho, the saturation its operator +publishes (`grid_provider_saturation_ratio`), and draws in proportion to its published +capacity (`grid_provider_capacity_requests`). It reads both when it orders the snapshot, +so a request only filters and draws. A site has room when rho is below 1. Among healthy +sites with room: + +- Three or more: two distinct sites are drawn in proportion to capacity, and the one + with the lower rho is chosen. The busiest of three is never chosen. +- Two: one draw, each site weighted by capacity over 1 + rho, with each share kept + between a tenth and nine tenths, so a recovering site is still measured. +- One: that site. + +With no site that has room, the gateway draws by capacity among the healthy sites tied +on the best polled queue depth. That also covers sites that publish no in-flight count +or capacity: a site without either has no load input, and sites without capacity are +drawn evenly. A cluster praxis reports with no healthy endpoint is never drawn while a +healthy one is left. + +A model sheds when every healthy site serving it has a fresh rho of at least 1.05. It +routes again once one reaches 0.95. A shed request gets 429 with `Retry-After` and an +OpenAI-style error, type `rate_limit_exceeded` and code `capacity_exhausted`, so a +client backs off the way it does for any overload. A model with no healthy routable +site gets 503 with code `no_healthy_site`: an outage, not load. The gateway keeps no +count of its own requests in flight. + ## Failure Behavior | Condition | Signal produced | Routing effect | diff --git a/docs/routing.md b/docs/routing.md index 348833d4..f3bbd7cd 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -2,6 +2,8 @@ AI Grid Network (AGN) routes inference traffic across provider gateways using a **multi-dimensional policy** rather than a single load-balancing algorithm. This guide starts with the routing outcome you want, then shows how policy, scoring, groups, affinity, and selection mode work together. +This guide covers the routing overlay and the consumer Praxis `intelligent_route` filter. A grid gateway serving `gridServing` chooses sites with `grid_site_route` instead, from load each site publishes: see [Tuning cross-site site selection](site-selection.md). + The practical model is: 1. Determine which providers are eligible to receive the request. diff --git a/docs/site-selection.md b/docs/site-selection.md new file mode 100644 index 00000000..253947bc --- /dev/null +++ b/docs/site-selection.md @@ -0,0 +1,132 @@ +# Tuning cross-site site selection + +This guide covers how a grid gateway serving `gridServing` chooses a site for each +request in `grid_site_route`, what feeds that choice, and what you can change. The +routing overlay and the consumer Praxis `intelligent_route` filter, with selection +groups and scoring policies, are a separate path described in +[routing.md](routing.md). + +## Site choice + +Each site's operator scrapes its EPP and publishes, per provider, how many requests +the provider holds and how many it can run at once. The gateway polls every site and +computes each site's load, rho, as requests held over capacity. A site has room when +rho is below 1. + +Among healthy sites with room: + +- **Three or more:** the gateway draws two sites in proportion to capacity and takes the + one with lower rho. +- **Two:** one draw, each site weighted by capacity over 1 + rho, each share kept between + 0.1 and 0.9. +- **One:** that site. + +With no site that has room, the gateway draws by capacity among the healthy sites tied +on the best polled queue depth. When every healthy site has rho of at least 1.05, the +model sheds: requests get 429 with `Retry-After` until a site reaches 0.95. + +A healthy site is ready and has a backend praxis reports healthy. A site that is not +ready, or whose cluster has no healthy endpoint, is chosen only when no healthy site is +left. + +## Inputs + +| Input | Published as | Source | +|---|---|---| +| Requests held | `grid_provider_in_flight_requests` | The larger of the EPP's `llm_d_epp_inflight_requests` summed over endpoints, and its average running plus queued requests times ready endpoints, plus requests its flow control holds (`llm_d_epp_flow_control_queue_size`) | +| Capacity | `grid_provider_capacity_requests` | `spec.maxRunning` times the EPP's fresh ready endpoints (`llm_d_epp_ready_endpoints`) | +| Load (rho) | `grid_provider_saturation_ratio` | Requests held over capacity, computed by the operator when both are known | +| Readiness | `grid_provider_ready` | 0 after two scrapes with no ready endpoints, or no successful scrape within `staleMetricsSeconds` | +| Queue depth | `llm_d_epp_average_queue_size` | The EPP, republished as is | + +Every input comes from the EPP's `/metrics`. Point `metricsConfig.metricsEndpoint` at +the EPP Service, since a pod address can reach a standby replica that reports nothing. +In-flight counts need the EPP's inflight-load-producer. Without it the pool averages +stand in, and they miss requests the EPP holds. + +A provider with no `maxRunning`, or no fresh endpoint, publishes no capacity. Its rho +is unknown, and it takes requests only through the queue-depth overflow. + +## Latency inputs + +Each operator also publishes recent latency per provider, over the last 30 seconds, +when at least 20 requests completed: `grid_provider_ttft_p50_seconds`, +`grid_provider_ttft_p90_seconds`, `grid_provider_tpot_seconds`, +`grid_provider_prefill_seconds_per_token`, and `grid_provider_error_ratio`. The gateway +reads them, but site choice does not use them: it ranks by rho alone. They show why one +site is slower than another, and the error ratio shows a failing site, which the latency +series would read as fast. + +The EPP times TTFT from receiving the request, so it includes flow-control wait and +network. Prefill seconds per token is an estimate that moves with the workload mix. + +## Choosing maxRunning + +Set `spec.maxRunning` to the most requests one endpoint runs at once, which is the +engine's `--max-num-seqs`. The operator multiplies it by ready endpoints, so it follows +scale-up and pod loss. + +An engine can run out of KV cache before it reaches max-num-seqs. vLLM logs its KV +concurrency at startup, as maximum concurrency for its configured tokens per request. +If your typical request is long, set `maxRunning` to the smaller of max-num-seqs and the +concurrency the KV cache holds at your typical request length. Too high a value makes a +KV-bound site look roomier than it is. + +On a prefill/decode pool, the EPP counts a request on both its prefill and its decode +endpoint, and ready endpoints count both roles. Use the same per-endpoint value for +both roles. + +## Settings + +| Setting | Default | Where | Effect of changing it | +|---|---|---|---| +| `spec.maxRunning` | none | InferenceProvider | Sets capacity per endpoint. Without it, rho is unknown. | +| `metricsConfig.metricsEndpoint` | none | InferenceProvider | The EPP Service to scrape. Required for every input. | +| `metricsConfig.staleMetricsSeconds` | half the signal TTL | InferenceProvider | How long a failing scrape keeps the last readiness before the provider reads not ready. | +| `GRID_SIGNALS_SCRAPE_INTERVAL_SECS` | 5 | Operator environment | How often the operator scrapes its EPP. Faster reacts sooner and loads the EPP more. | +| `metricsListener.enabled` | false | praxis-gateway chart | Serves the decision counters on a TLS listener, with a Service and ServiceMonitor. | + +The rest is fixed: + +- The gateway polls each site every 5s with a 2s timeout. A load sample is fresh for + two polls, 10s. +- Shedding starts when every healthy site holds rho of at least 1.05 and stops when one + reaches 0.95. +- A two-site draw keeps each share between 0.1 and 0.9. +- One request weighs at most 64 sites with room. +- A shed 429 and an outage 503 each carry a Retry-After of 3 to 7 seconds. + +## What to watch + +- `grid_route_decisions_total{site,reason}` on the gateway's metrics listener. `routed` + and `fallback` count by site. Refusals count under an empty site as `not_ready`, + `no_route`, `bad_request`, or `shed`. +- `grid_route_site_score{site,cluster}`: the queue depth each candidate was ordered by. + NaN marks an excluded or demoted candidate. +- `grid_provider_in_flight_requests`, `grid_provider_capacity_requests`, + `grid_provider_saturation_ratio`, and `grid_provider_ready` on each site's signals + endpoint, `/v1/site/signals`, and on each operator's `/metrics`. A hub's `/metrics` + also carries what it polled from its peers. +- `x-grid-site` and `x-grid-backend` on every routed response. +- Operator WARN `serving config: refusing candidates whose site is not a DNS-1123 label`. + +## Symptoms + +| Symptom | Likely cause | Action | +|---|---|---| +| Traffic piles onto one site | No capacity published, so the queue-depth overflow herds onto the site tied on the best queue for one poll window | Set `maxRunning` on every provider and check `grid_provider_capacity_requests` appears for each | +| A site is never chosen | Not ready, or its cluster has no healthy endpoint. Its `grid_route_site_score` is NaN. | Check `grid_provider_ready` and the gateway's cluster health log line. | +| A small or slow site takes little | Expected: draws follow capacity, and two choices never pick the busiest of three | Raise its `maxRunning` only if the engine runs more | +| 429 `capacity_exhausted` under light load | rho reads high: `maxRunning` too low, or in-flight counts both P/D roles against a decode-only capacity | Compare `grid_provider_in_flight_requests` with what the engines run, and correct `maxRunning` | +| 503 `no_healthy_site` | Every site serving the model is not ready or has no route from this gateway | Check `grid_provider_ready` and the serving config candidates | +| Traffic moves between sites every few seconds | Load near capacity, read at 5s polls | Expected near full. Check capacity covers the load. | +| A peer never routes | Its site id is not a DNS-1123 label, and the operator refuses it | Look for the refused-site WARN and rename the site | + +## What you cannot tune + +The gateway keeps no count of its own requests. Load comes from what each site's +operator publishes, so it is as fresh as the scrape and poll, about 5 to 15 seconds. The +selection is coarse by design. It keeps each site near its capacity share and away from +full, and the site's EPP picks the endpoint within the site. No per-site weight or +threshold is settable. + diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 9bf30a3b..5cade8a9 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -36,6 +36,7 @@ dependencies = [ "grid-signals-client", "http", "metrics", + "praxis-proxy-core", "praxis-proxy-filter", "praxis-proxy-tls", "rustls", @@ -43,6 +44,7 @@ dependencies = [ "tokio", "tokio-rustls", "tracing", + "tracing-subscriber", "yaml_serde", "zeroize", ] @@ -1381,12 +1383,18 @@ version = "0.1.4" dependencies = [ "ai-grid-filters", "certs", + "httparse", "praxis-ai-filters", "praxis-proxy", "praxis-proxy-core", "praxis-proxy-filter", + "praxis-proxy-protocol", + "praxis-proxy-tls", "rustls", + "tokio", + "tokio-rustls", "tracing", + "yaml_serde", ] [[package]] diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index e36c2b44..2f8efaf7 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -314,11 +314,18 @@ praxis-proxy-core = "0.7.3" praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false } ai-grid-filters = { path = "ai-grid-filters" } tracing = "0.1.44" +praxis-protocol = { version = "0.7.3", package = "praxis-proxy-protocol" } +praxis-tls = { version = "0.7.3", package = "praxis-proxy-tls" } +tokio = { version = "1", features = ["io-util", "net", "rt", "sync", "time"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } +httparse = "1.10" [dev-dependencies] # The reload test runs a real mTLS signals peer against the binary. certs = { path = "../certs", default-features = false, features = ["verifier"] } rustls = { version = "0.23.45", features = ["ring"] } +serde_yaml = { version = "0.10", package = "yaml_serde" } +tokio = { version = "1", features = ["macros"] } # machete can't see these: used as `praxis` / `praxis_core` / `praxis_filter`, not their package names. [package.metadata.cargo-machete] diff --git a/gateway/ai-grid-filters/Cargo.toml b/gateway/ai-grid-filters/Cargo.toml index bade9a45..7586948d 100644 --- a/gateway/ai-grid-filters/Cargo.toml +++ b/gateway/ai-grid-filters/Cargo.toml @@ -8,7 +8,9 @@ license.workspace = true publish = false [dependencies] -praxis-filter = { version = "0.7.0", package = "praxis-proxy-filter" } +praxis-filter = { version = "0.7.1", package = "praxis-proxy-filter" } +praxis-core = { version = "0.7.1", package = "praxis-proxy-core" } +praxis-tls = { version = "0.7.1", package = "praxis-proxy-tls" } grid-signals = { path = "../../signals" } # The gateway wiring spawns the pollers and builds their mTLS scrapers, so the # poller and cert issuance are real dependencies, not just test ones. @@ -25,6 +27,7 @@ http = "1" rustls = { version = "0.23", default-features = false, features = ["logging", "std", "tls12"] } metrics = "0.24.6" zeroize = "1.9.0" +tokio = { version = "1", features = ["net", "rt", "time"] } [dev-dependencies] # The flow tests bind a mock TLS peer to drive the real scrape path. @@ -32,6 +35,7 @@ tokio = { version = "1", features = ["macros", "net", "rt", "rt-multi-thread", " tokio-rustls = "0.26" rustls = { version = "0.23", features = ["ring"] } praxis-tls = { version = "0.7.1", package = "praxis-proxy-tls" } +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "std"] } [lints] workspace = true diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index ce94978d..1c913dcd 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -18,8 +18,9 @@ use praxis_filter::FilterError; use crate::{ descriptor::{RouteCandidate, validate_candidates, validate_local_site}, + health::ClusterHealth, serving::{GridServingConfig, PeerServingConfig, validate_peer}, - snapshot::RouteSnapshot, + snapshot::{Gauged, RouteSnapshot}, }; /// The order step a poller runs after each scrape. @@ -45,22 +46,43 @@ impl Topology { /// Validate the topology half of `config`. fn from_config(config: &GridServingConfig) -> Result { validate_local_site(&config.local_site)?; + let base = validate_candidates(config.candidates.clone())?; Ok(Self { - base: Arc::from(validate_candidates(config.candidates.clone())?), + base: Arc::from(base), local_site: Arc::from(config.local_site.as_str()), load_window_ms: config.load_window_ms, }) } - /// The snapshot ordered from `store` at `now`. - fn order(&self, store: &LoadStore, now: i64) -> RouteSnapshot { - RouteSnapshot::from_store( + /// The snapshot ordered from `store` at `now`, down clusters and peer gateways last, with the + /// models to shed given the set `shedding` the previous snapshot shed. + #[expect( + clippy::too_many_arguments, + reason = "each input is a distinct piece of control state" + )] + fn order( + &self, + store: &LoadStore, + now: i64, + health: &ClusterHealth, + shedding: &std::collections::BTreeSet>, + gauged: &mut Gauged, + ) -> RouteSnapshot { + let ordered = RouteSnapshot::from_store( self.base.iter().cloned().collect(), Arc::clone(&self.local_site), store, now, self.load_window_ms, - ) + ); + // Praxis demotes a cluster only once it has reported health. Without a registry the + // gateway knows nothing about backends, so it demotes nothing rather than guessing. + let ordered = if health.observed() { + ordered.demote(&health.down()) + } else { + ordered + }; + ordered.shed(shedding).published(gauged) } } @@ -110,8 +132,9 @@ pub(crate) struct Control { /// The topology every refresh orders, swapped on reload. topology: Arc>, - /// Serializes snapshot stores between the refreshes and a reload. - swap: Arc>, + /// Serializes snapshot stores between the refreshes and a reload, and holds the + /// site/cluster pairs with a published score. + swap: Arc>, /// Store retention, fixed for the process. window_secs: u64, @@ -127,6 +150,9 @@ pub(crate) struct Control { /// Builds and spawns a peer's poller. start: StartPeer, + + /// Backend clusters with no healthy endpoint, ordered last on every refresh. + health: Arc, } impl Control { @@ -134,23 +160,36 @@ impl Control { pub(crate) fn new(config: &GridServingConfig, start: StartPeer) -> Result { let topology = Topology::from_config(config)?; // Cold start: config order until the first poll re-orders it by live load. + let mut gauged = Gauged::new(); let cold_start = RouteSnapshot::from_static( topology.base.iter().cloned().collect(), Arc::clone(&topology.local_site), - ); + ) + .published(&mut gauged); Ok(Self { store: Arc::new(LoadStore::new(Duration::from_secs(config.window_secs))), snapshot: Arc::new(ArcSwap::from_pointee(cold_start)), topology: Arc::new(ArcSwap::from_pointee(topology)), - swap: Arc::new(Mutex::new(())), + swap: Arc::new(Mutex::new(gauged)), window_secs: config.window_secs, peers: HashMap::new(), applied: None, identity: None, start, + health: Arc::default(), }) } + /// The cluster health the route filter publishes into and every refresh reads. + pub(crate) fn health(&self) -> Arc { + Arc::clone(&self.health) + } + + /// The live load store. + pub(crate) fn store(&self) -> Arc { + Arc::clone(&self.store) + } + /// The snapshot the filter reads. pub(crate) fn snapshot(&self) -> Arc> { Arc::clone(&self.snapshot) @@ -176,7 +215,9 @@ impl Control { return Ok(None); } // Validated before any poller starts, so an invalid config starts and drops nothing. - let topology = Arc::new(validate_config(config)?); + validate_config(config)?; + // Built after validation: it reads the identity from disk, so a failure here is retried, never settled. + let topology = Arc::new(Topology::from_config(config)?); if config.window_secs != self.window_secs { tracing::warn!( current = self.window_secs, @@ -186,11 +227,14 @@ impl Control { } let (next, started) = self.start_changed(config, renewed)?; let outcome = self.reconcile(&next, started, renewed, config.load_window_ms); - let ordered = Arc::new(topology.order(&self.store, now_ms())); { - let _swapping = self.swap.lock().unwrap_or_else(PoisonError::into_inner); + // Order under the lock, so a concurrent refresh never publishes an older down set or load. + let mut gauged = self.swap.lock().unwrap_or_else(PoisonError::into_inner); + let shedding = self.snapshot.load().shedding.clone(); + let ordered = Arc::new(topology.order(&self.store, now_ms(), &self.health, &shedding, &mut gauged)); self.topology.store(topology); self.snapshot.store(ordered); + drop(gauged); } // The reload stands and its topology is published: only now may the new pollers write, // so their first refresh orders the new topology. Committing a kept poller is a no-op. @@ -271,19 +315,21 @@ impl Control { } /// The refresh a poller runs each cycle: order the current topology by load. - fn refresh(&self) -> Refresh { + pub(crate) fn refresh(&self) -> Refresh { make_refresh( Arc::clone(&self.topology), Arc::clone(&self.snapshot), Arc::clone(&self.swap), + Arc::clone(&self.health), now_ms, ) } } /// The topology of `config`, refusing what no retry can fix: a bad candidate, peer, or duplicate site. -fn validate_config(config: &GridServingConfig) -> Result { - let topology = Topology::from_config(config)?; +fn validate_config(config: &GridServingConfig) -> Result<(), FilterError> { + validate_local_site(&config.local_site)?; + validate_candidates(config.candidates.clone())?; let mut sites = std::collections::HashSet::with_capacity(config.peers.len()); for peer in &config.peers { validate_peer(peer)?; @@ -291,52 +337,37 @@ fn validate_config(config: &GridServingConfig) -> Result return Err(format!("grid: peer site {} appears twice", peer.site).into()); } } - Ok(topology) + Ok(()) } -/// Order the current topology and swap it in, unless a reload replaced it meanwhile. +/// Order the current topology and swap it in. +/// +/// Every refresher orders under `swap`, so the last store always reflects the latest +/// topology, down set, and load. pub(crate) fn make_refresh( topology: Arc>, snapshot: Arc>, - swap: Arc>, + swap: Arc>, + health: Arc, now: N, ) -> Refresh where N: Fn() -> i64 + Send + 'static, { Box::new(move |store: &LoadStore| { - let current = topology.load_full(); - let ordered = Arc::new(current.order(store, now())); - let _swapping = swap.lock().unwrap_or_else(PoisonError::into_inner); - if Arc::ptr_eq(¤t, &topology.load()) { - snapshot.store(ordered); - } + let mut gauged = swap.lock().unwrap_or_else(PoisonError::into_inner); + let shedding = snapshot.load().shedding.clone(); + let ordered = topology.load().order(store, now(), &health, &shedding, &mut gauged); + snapshot.store(Arc::new(ordered)); + drop(gauged); }) } /// A change detector over every identity file `config` names: the grid CA, /// client certificate, and key. Not a security function. /// -/// A rotation swaps the files at once but they are read one by one, so a pass -/// that straddles the swap mixes versions. Read until two passes agree. +/// Each file is hashed on its own, so no buffer holds the concatenated key. fn identity_digest(config: &GridServingConfig) -> [u8; 32] { - let mut last = read_identity(config); - for _ in 0..IDENTITY_READS { - let next = read_identity(config); - if next == last { - break; - } - last = next; - } - last -} - -/// Passes [`identity_digest`] makes after the first; a rotation tears at most one. -const IDENTITY_READS: usize = 3; - -/// One pass over the identity files. Each file is hashed on its own, so no -/// buffer holds the concatenated key. -fn read_identity(config: &GridServingConfig) -> [u8; 32] { let paths: std::collections::BTreeSet<&str> = config .peers .iter() @@ -448,7 +479,7 @@ pub(crate) fn watch(control: Arc>, path: PathBuf, every: Duration fn apply_file(control: &Mutex, path: &std::path::Path, bytes: &[u8], tally: &WatchCounts) -> bool { let parsed = serde_yaml::from_slice::(bytes) .map_err(|error| -> FilterError { format!("grid: parsing {}: {error}", path.display()).into() }) - .and_then(|config| validate_config(&config).map(|_| config)); + .and_then(|config| validate_config(&config).map(|()| config)); let (result, settled) = match parsed { Ok(config) => { let result = control.lock().unwrap_or_else(PoisonError::into_inner).apply(&config); @@ -493,7 +524,8 @@ fn report(result: &Result, FilterError>, tally: &WatchCoun kept = outcome.kept, "{applied}" ), - Ok(None) => tracing::info!("grid: serving config unchanged; peer pollers reused"), + // Nothing changed, so nothing to say at info: the counter still records the attempt. + Ok(None) => tracing::debug!("grid: serving config unchanged; peer pollers reused"), Err(error) => tracing::warn!(%error, "grid: serving config rejected; keeping the last good config"), } } @@ -508,7 +540,10 @@ fn report(result: &Result, FilterError>, tally: &WatchCoun reason = "tests; the pollers run on their own threads, so these sync tests wait with thread::sleep" )] mod tests { - use std::{sync::atomic::AtomicI64, time::Instant}; + use std::{ + sync::atomic::{AtomicBool, AtomicI64}, + time::Instant, + }; use grid_signals_client::{FetchError, Scrape, SignalSource, spawn_on_thread_held}; @@ -597,6 +632,7 @@ mod tests { fn candidate(site: &str) -> CandidateConfig { CandidateConfig { + admission: crate::descriptor::AdmissionState::default(), cluster: format!("pool-{site}"), credential: None, fresh: true, @@ -620,6 +656,7 @@ mod tests { client_cert_path: "/etc/grid/tls.crt".to_owned(), client_key_path: "/etc/grid/tls.key".to_owned(), pins: Vec::new(), + gateway: None, } } @@ -638,58 +675,6 @@ mod tests { snapshot.candidates.iter().map(|c| c.site.to_string()).collect() } - /// The routable sites, sorted: which sites route, not their load order. - fn members(snapshot: &RouteSnapshot) -> Vec { - let mut sites = sites(snapshot); - sites.sort(); - sites - } - - /// A `Secret` or `ConfigMap` volume: each file resolves through `..data`, which - /// `write` swaps in one rename, as the kubelet does. - struct Mount { - dir: PathBuf, - generation: usize, - } - - impl Mount { - fn new(name: &str, files: &[(&str, &str)]) -> Self { - let dir = std::env::temp_dir().join(format!("{name}-{}", std::process::id())); - let _stale = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).expect("dir"); - let mut mount = Self { dir, generation: 0 }; - mount.write(files); - mount - } - - fn path(&self, name: &str) -> String { - self.dir.join(name).to_string_lossy().into_owned() - } - - /// Publish `files` together; earlier files are dropped, as in a new Secret version. - fn write(&mut self, files: &[(&str, &str)]) { - self.generation = self.generation.saturating_add(1); - let version = format!("..{}", self.generation); - std::fs::create_dir(self.dir.join(&version)).expect("version dir"); - for (name, content) in files { - std::fs::write(self.dir.join(&version).join(name), content).expect("write"); - let link = self.dir.join(name); - if std::fs::symlink_metadata(&link).is_err() { - std::os::unix::fs::symlink(format!("..data/{name}"), &link).expect("file link"); - } - } - let staged = self.dir.join("..data_tmp"); - std::os::unix::fs::symlink(&version, &staged).expect("data link"); - std::fs::rename(&staged, self.dir.join("..data")).expect("swap"); - } - } - - impl Drop for Mount { - fn drop(&mut self) { - let _removed = std::fs::remove_dir_all(&self.dir); - } - } - fn front(snapshot: &ArcSwap) -> Option { snapshot.load().candidates.first().map(|c| c.site.to_string()) } @@ -810,10 +795,8 @@ mod tests { #[test] fn an_invalid_config_keeps_the_last_good_one() { let peers = Peers::default(); - let mut control = Control::new(&config(&["east"]), peers.starter()).expect("control"); - control.apply(&config(&["east"])).expect("first apply"); - // The topology, not the snapshot: a poller's scrape re-publishes the snapshot. - let before = control.topology.load_full(); + let grid = runtime(&peers, &config(&["east"])); + let before = grid.snapshot().load_full(); let mut bad_candidate = config(&["east", "west"]); bad_candidate.candidates[1].name = String::new(); @@ -822,17 +805,14 @@ mod tests { let mut zero = config(&["east", "west"]); zero.peers[1].interval_ms = 0; for bad in [bad_candidate, twice, zero] { - control.apply(&bad).expect_err("an invalid config is rejected"); + grid.reload(&bad).expect_err("an invalid config is rejected"); } - assert!( - Arc::ptr_eq(&before, &control.topology.load_full()), - "a rejected config publishes no topology" - ); + // Content, not identity: a poll may republish the same order at any time. assert_eq!( - sites(&control.snapshot().load()), - ["east"], - "the routable sites are unchanged" + sites(&before), + sites(&grid.snapshot().load_full()), + "the snapshot keeps the last good topology" ); assert_eq!(peers.starts("west"), 0, "no poller started for a rejected config"); assert_eq!(peers.starts("east"), 1); @@ -1037,36 +1017,43 @@ mod tests { grid.reload(&config(&["east"])).expect("reload"); - assert_eq!(members(&in_flight), ["east", "west"], "the request keeps its snapshot"); + // A poller refresh may reorder the snapshot, so compare the set. + let mut kept = sites(&in_flight); + kept.sort(); + assert_eq!(kept, ["east", "west"], "the request keeps its snapshot"); assert_eq!(sites(&snapshot.load()), ["east"], "the next request sees the new one"); } #[test] - fn a_refresh_that_raced_a_reload_is_dropped() { - let first = Arc::new(ArcSwap::from_pointee( + fn a_refresh_orders_while_holding_the_swap_lock() { + // Ordering under the lock means a reload or health tick cannot publish between + // this cycle reading its inputs and storing its order. + let topology = Arc::new(ArcSwap::from_pointee( Topology::from_config(&config(&["east"])).expect("topology"), )); let snapshot = Arc::new(ArcSwap::from_pointee(RouteSnapshot::from_static( Vec::new(), Arc::from("local"), ))); - let reloaded = Arc::new(Topology::from_config(&config(&["west"])).expect("topology")); + let swap = Arc::new(Mutex::new(Gauged::new())); + let held = Arc::new(AtomicBool::new(false)); let refresh = { - let topology = Arc::clone(&first); + let (swap, held) = (Arc::clone(&swap), Arc::clone(&held)); make_refresh( - Arc::clone(&first), + topology, Arc::clone(&snapshot), - Arc::new(Mutex::new(())), + Arc::clone(&swap), + Arc::default(), move || { - // The reload lands while this cycle is ordering. - topology.store(Arc::clone(&reloaded)); + held.store(swap.try_lock().is_err(), Ordering::SeqCst); 1_000 }, ) }; refresh(&LoadStore::new(Duration::from_secs(60))); - assert!(sites(&snapshot.load()).is_empty(), "the stale order is not stored"); + assert!(held.load(Ordering::SeqCst), "ordered outside the swap lock"); + assert_eq!(sites(&snapshot.load()), ["east"]); } #[test] @@ -1081,9 +1068,13 @@ mod tests { let clock = Arc::new(AtomicI64::new(1_000)); let refresh = { let clock = Arc::clone(&clock); - make_refresh(topology, Arc::clone(&snapshot), Arc::new(Mutex::new(())), move || { - clock.load(Ordering::SeqCst) - }) + make_refresh( + topology, + Arc::clone(&snapshot), + Arc::new(Mutex::new(Gauged::new())), + Arc::default(), + move || clock.load(Ordering::SeqCst), + ) }; let line = |site: &str, value: f64, at: i64| { format!(r#"{LOAD_METRIC}{{grid_site="{site}",grid_provider="pool-{site}"}} {value} {at}"#) @@ -1136,7 +1127,7 @@ mod tests { write(&yaml(&["east", "west"])); eventually("the rewrite handled", || counts().applied() == 1); - assert_eq!(members(&snapshot.load()), ["east", "west"], "the rewrite applied"); + assert_eq!(sites(&snapshot.load()), ["east", "west"], "the rewrite applied"); eventually("west polled", || peers.fetches("west") > 0); drop(grid); @@ -1145,17 +1136,18 @@ mod tests { #[test] fn a_renewed_identity_restarts_the_pollers_with_no_config_change() { - let mut identity = Mount::new( - "grid-identity", - &[("ca.pem", "old"), ("tls.crt", "old"), ("tls.key", "old")], - ); + let dir = std::env::temp_dir().join(format!("grid-identity-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let file = |name: &str| dir.join(name).to_string_lossy().into_owned(); + for name in ["ca.pem", "tls.crt", "tls.key"] { + std::fs::write(file(name), "old").expect("write"); + } let mut serving = config(&["east"]); - serving.peers[0].grid_ca_path = identity.path("ca.pem"); - serving.peers[0].client_cert_path = identity.path("tls.crt"); - serving.peers[0].client_key_path = identity.path("tls.key"); - let yaml = serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml"); - let file = Mount::new("grid-identity-serving", &[("serving.yaml", &yaml)]); - let path = file.path("serving.yaml"); + serving.peers[0].grid_ca_path = file("ca.pem"); + serving.peers[0].client_cert_path = file("tls.crt"); + serving.peers[0].client_key_path = file("tls.key"); + let path = dir.join("serving.yaml"); + std::fs::write(&path, serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml")).expect("write"); let peers = Peers::default(); let mut grid = runtime(&peers, &serving); @@ -1168,59 +1160,23 @@ mod tests { let counts = || grid.watcher().expect("watching").counts(); eventually("the startup file seen", || counts().reused() == 1); - identity.write(&[("ca.pem", "old"), ("tls.crt", "renewed"), ("tls.key", "renewed")]); + for name in ["tls.crt", "tls.key"] { + std::fs::write(file(name), "renewed").expect("renew"); + } eventually("the renewal applied", || counts().applied() == 1); assert_eq!(peers.starts("east"), 2, "the poller restarted on the renewed identity"); std::thread::sleep(Duration::from_millis(100)); assert_eq!(counts().applied(), 1, "a settled identity restarts nothing more"); drop(grid); - } - - #[test] - fn an_identity_read_during_a_rotation_sees_one_version() { - let old = [("ca.pem", "ca"), ("tls.crt", "old"), ("tls.key", "old")]; - let new = [("ca.pem", "ca"), ("tls.crt", "new"), ("tls.key", "new")]; - let mut identity = Mount::new("grid-torn", &old); - let mut serving = config(&["east"]); - serving.peers[0].grid_ca_path = identity.path("ca.pem"); - serving.peers[0].client_cert_path = identity.path("tls.crt"); - serving.peers[0].client_key_path = identity.path("tls.key"); - let old_digest = identity_digest(&serving); - identity.write(&new); - let new_digest = identity_digest(&serving); - - // Each swap waits for two reads after the last, so no read spans two swaps, - // however long the reader stalls. A real rotation is hours apart. - let reads = Arc::new(AtomicUsize::new(0)); - let counted = Arc::clone(&reads); - let rotating = std::thread::spawn(move || { - for round in 0..40 { - let after = counted.load(Ordering::SeqCst).saturating_add(2); - let deadline = Instant::now().checked_add(Duration::from_secs(5)).expect("deadline"); - while counted.load(Ordering::SeqCst) < after && Instant::now() < deadline { - std::thread::yield_now(); - } - identity.write(if round % 2 == 0 { &old } else { &new }); - } - identity - }); - while !rotating.is_finished() { - let digest = identity_digest(&serving); - assert!( - digest == old_digest || digest == new_digest, - "read {} mixed two versions of the identity", - reads.load(Ordering::SeqCst) - ); - reads.fetch_add(1, Ordering::SeqCst); - } - drop(rotating.join().expect("rotation")); + std::fs::remove_dir_all(&dir).expect("cleanup"); } #[test] fn a_change_that_fails_on_unreadable_identity_is_retried_until_it_applies() { - let mut identity = Mount::new("grid-retry", &[]); - let cert = PathBuf::from(identity.path("tls.crt")); + let dir = std::env::temp_dir().join(format!("grid-retry-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let cert = dir.join("tls.crt"); let peers = Peers::default(); let ok = peers.starter(); let watched = cert.clone(); @@ -1233,39 +1189,39 @@ mod tests { let mut grid = crate::serving::start_runtime(&config(&["west"]), start).expect("runtime starts"); let mut serving = config(&["west", "east"]); serving.peers[1].client_cert_path = cert.to_string_lossy().into_owned(); - let yaml = serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml"); - let file = Mount::new("grid-retry-serving", &[("serving.yaml", &yaml)]); - let path = file.path("serving.yaml"); + let path = dir.join("serving.yaml"); + std::fs::write(&path, serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml")).expect("write"); grid.watch(&path, Duration::from_millis(20)).expect("watch"); let counts = || grid.watcher().expect("watching").counts(); eventually("the change retried", || counts().rejected() >= 2); assert_eq!(sites(&grid.snapshot().load()), ["west"], "the old config stays"); - identity.write(&[("tls.crt", "issued")]); + std::fs::write(&cert, "issued").expect("issue"); eventually("the pending change applied", || counts().applied() == 1); - assert_eq!( - members(&grid.snapshot().load()), - ["east", "west"], - "the pending change applied" - ); + assert_eq!(sites(&grid.snapshot().load()), ["west", "east"]); drop(grid); + std::fs::remove_dir_all(&dir).expect("cleanup"); } #[test] fn a_refused_file_neither_blocks_renewal_nor_repeats_its_rejection() { - let mut identity = Mount::new( - "grid-refused", - &[("ca.pem", "old"), ("tls.crt", "old"), ("tls.key", "old")], - ); + let dir = std::env::temp_dir().join(format!("grid-refused-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let file = |name: &str| dir.join(name).to_string_lossy().into_owned(); + for name in ["ca.pem", "tls.crt", "tls.key"] { + std::fs::write(file(name), "old").expect("write"); + } let mut serving = config(&["east"]); - serving.peers[0].grid_ca_path = identity.path("ca.pem"); - serving.peers[0].client_cert_path = identity.path("tls.crt"); - serving.peers[0].client_key_path = identity.path("tls.key"); - let yaml = |config: &GridServingConfig| serde_yaml::to_string(&serde_yaml_value(config)).expect("yaml"); - let mut file = Mount::new("grid-refused-serving", &[("serving.yaml", &yaml(&serving))]); - let path = file.path("serving.yaml"); + serving.peers[0].grid_ca_path = file("ca.pem"); + serving.peers[0].client_cert_path = file("tls.crt"); + serving.peers[0].client_key_path = file("tls.key"); + let path = dir.join("serving.yaml"); + let write = |config: &GridServingConfig| { + std::fs::write(&path, serde_yaml::to_string(&serde_yaml_value(config)).expect("yaml")).expect("write"); + }; + write(&serving); let peers = Peers::default(); let mut grid = runtime(&peers, &serving); @@ -1275,7 +1231,7 @@ mod tests { let mut twice = config(&["east", "west"]); twice.peers[1].site = "east".to_owned(); - file.write(&[("serving.yaml", &yaml(&twice))]); + write(&twice); eventually("the duplicate site refused", || counts().rejected() == 1); std::thread::sleep(Duration::from_millis(200)); assert_eq!( @@ -1285,7 +1241,9 @@ mod tests { ); assert_eq!(peers.starts("east"), 1, "a refused file starts and drops no poller"); - identity.write(&[("ca.pem", "old"), ("tls.crt", "renewed"), ("tls.key", "renewed")]); + for name in ["tls.crt", "tls.key"] { + std::fs::write(file(name), "renewed").expect("renew"); + } eventually("the running config renewed", || counts().applied() == 1); assert_eq!( peers.starts("east"), @@ -1299,6 +1257,7 @@ mod tests { ); drop(grid); + std::fs::remove_dir_all(&dir).expect("cleanup"); } /// `config` in the operator's serving-config shape. @@ -1307,11 +1266,15 @@ mod tests { .peers .iter() .map(|p| { - serde_yaml::from_str(&format!( + let mut peer: serde_yaml::Value = serde_yaml::from_str(&format!( "{{site: {}, addr: '{}', server_name: {}, authority: {}, grid_ca_path: {}, client_cert_path: {}, client_key_path: {}}}", p.site, p.addr, p.server_name, p.authority, p.grid_ca_path, p.client_cert_path, p.client_key_path )) - .expect("peer") + .expect("peer"); + if let (Some(gateway), Some(fields)) = (&p.gateway, peer.as_mapping_mut()) { + fields.insert("gateway".into(), gateway.clone().into()); + } + peer }) .collect(); let candidates: Vec = config @@ -1333,4 +1296,61 @@ mod tests { root.insert("peers".into(), peers.into()); root.into() } + + /// A writer the test subscriber logs into. + #[derive(Clone, Default)] + struct Captured(Arc>>); + + impl std::io::Write for Captured { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().expect("log buffer").extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for Captured { + type Writer = Self; + + fn make_writer(&'writer self) -> Self::Writer { + self.clone() + } + } + + /// What `run` logs at info and above. + fn info_log(run: impl FnOnce()) -> String { + let out = Captured::default(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .with_writer(out.clone()) + .finish(); + tracing::subscriber::with_default(subscriber, run); + String::from_utf8(out.0.lock().expect("log buffer").clone()).expect("utf-8 log") + } + + #[test] + fn unchanged_reload_logs_nothing_at_info() { + let tally = WatchCounts::default(); + let quiet = info_log(|| report(&Ok(None), &tally, "grid: serving config reloaded")); + assert!(quiet.is_empty(), "a no-op reload logged at info: {quiet}"); + assert_eq!( + tally.reused.load(Ordering::SeqCst), + 1, + "the no-op reload is still counted" + ); + + let outcome = ReloadOutcome { + started: 1, + ..ReloadOutcome::default() + }; + let loud = info_log(|| report(&Ok(Some(outcome)), &tally, "grid: serving config reloaded")); + assert!( + loud.contains("grid: serving config reloaded"), + "a real reload logs at info: {loud}" + ); + } } diff --git a/gateway/ai-grid-filters/src/decisions.rs b/gateway/ai-grid-filters/src/decisions.rs new file mode 100644 index 00000000..556c7432 --- /dev/null +++ b/gateway/ai-grid-filters/src/decisions.rs @@ -0,0 +1,133 @@ +//! `grid_route_decisions_total`: where `grid_site_route` sent a request, or why it refused. +//! +//! Counter handles are registered off the request path, so recording is one atomic add. + +use std::sync::{Arc, LazyLock}; + +use metrics::Counter; + +/// The counter's name. +const METRIC: &str = "grid_route_decisions_total"; + +/// The `reason` label: a closed set, so the series count is fixed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Outcome { + /// Sent to a healthy site. How it ranked is in `grid_route_site_score`. + Routed, + /// Sent to a demoted site because no healthy one was left. + Fallback, + /// Every candidate was excluded. Answered 503. + NotReady, + /// An admitted candidate had no route from this gateway. Answered 503. + NoRoute, + /// The request named no model, or one no candidate serves. Answered 400 or 404. + BadRequest, + /// Every healthy site serving the model was past full. Answered 503. + Shed, +} + +impl Outcome { + /// Every value, in label order. + #[cfg(test)] + pub(crate) const ALL: [Self; 6] = [ + Self::Routed, + Self::Fallback, + Self::NotReady, + Self::NoRoute, + Self::BadRequest, + Self::Shed, + ]; + + /// The `reason` label value. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Routed => "routed", + Self::Fallback => "fallback", + Self::NotReady => "not_ready", + Self::NoRoute => "no_route", + Self::BadRequest => "bad_request", + Self::Shed => "shed", + } + } +} + +/// Why a request went nowhere. It picks the status, and its [`Outcome`] is the label. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Refused { + /// Every match was excluded, by readiness or admission. + NotReady, + /// An admitted match existed but had no route from this gateway. + NoRoute, + /// No candidate serves the model. + UnknownModel, + /// The request named no model. + NoModel, + /// Every healthy site serving the model was past full. + Shed, +} + +impl Refused { + /// The label this refusal counts under. + pub(crate) fn outcome(self) -> Outcome { + match self { + Self::NotReady => Outcome::NotReady, + Self::NoRoute => Outcome::NoRoute, + Self::UnknownModel | Self::NoModel => Outcome::BadRequest, + Self::Shed => Outcome::Shed, + } + } + + /// Count one refused request, under site "". + pub(crate) fn record(self) { + // Built on first use, after the gateway installs the recorder. + static REFUSED: LazyLock<[Counter; 4]> = LazyLock::new(|| { + [Outcome::NotReady, Outcome::NoRoute, Outcome::BadRequest, Outcome::Shed] + .map(|outcome| metrics::counter!(METRIC, "site" => "", "reason" => outcome.as_str())) + }); + let index = match self.outcome() { + Outcome::NotReady => 0, + Outcome::NoRoute => 1, + Outcome::BadRequest | Outcome::Routed | Outcome::Fallback => 2, + Outcome::Shed => 3, + }; + if let Some(counter) = REFUSED.get(index) { + counter.increment(1); + } + } +} + +/// One site's counters: routed, then fallback. +#[derive(Debug)] +pub struct SiteDecisions([Counter; 2]); + +impl SiteDecisions { + /// Register the counters for `site`, a name from the serving config. + pub(crate) fn new(site: &Arc) -> Self { + Self( + [Outcome::Routed, Outcome::Fallback] + .map(|outcome| metrics::counter!(METRIC, "site" => Arc::clone(site), "reason" => outcome.as_str())), + ) + } + + /// Count one request sent to this site, as a fallback when no healthy site was left. + pub(crate) fn record(&self, fallback: bool) { + if let Some(counter) = self.0.get(usize::from(fallback)) { + counter.increment(1); + } + } +} + +#[cfg(test)] +mod tests { + use super::Outcome; + + #[test] + fn the_reason_label_is_a_closed_set() { + let labels = Outcome::ALL.map(Outcome::as_str); + assert_eq!( + labels, + ["routed", "fallback", "not_ready", "no_route", "bad_request", "shed"], + "adding a reason is a deliberate change: update the chart README metrics section" + ); + } +} diff --git a/gateway/ai-grid-filters/src/descriptor.rs b/gateway/ai-grid-filters/src/descriptor.rs index 73972938..b7de80f4 100644 --- a/gateway/ai-grid-filters/src/descriptor.rs +++ b/gateway/ai-grid-filters/src/descriptor.rs @@ -7,6 +7,7 @@ use std::{collections::HashSet, sync::Arc}; +use praxis_core::connectivity::Upstream; use praxis_filter::FilterError; use serde::Deserialize; @@ -50,7 +51,8 @@ impl CapabilityKind { /// /// Controls whether a candidate accepts new sessions, existing sessions only, /// or is excluded from routing entirely. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] pub enum AdmissionState { /// Accepts both new and existing sessions. #[default] @@ -59,7 +61,8 @@ pub enum AdmissionState { /// Accepts only existing sessions (bound via session affinity). ExistingOnly, - /// Excluded from routing entirely. + /// Excluded from routing entirely. `none` on the wire, as the operator writes it. + #[serde(rename = "none")] Excluded, } @@ -86,6 +89,10 @@ impl AdmissionState { #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] #[serde(deny_unknown_fields)] pub struct CandidateConfig { + /// Whether it takes new requests, as the operator resolved it. Absent means it does. + #[serde(default)] + pub admission: AdmissionState, + /// Cluster name to select when this candidate is chosen. pub cluster: String, @@ -112,19 +119,43 @@ fn default_fresh() -> bool { true } +/// A provider's recent latency, as its site's operator publishes it. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Latency { + /// Median streaming time to first token, seconds. + pub ttft_p50: Option, + /// 90th percentile streaming time to first token, seconds. + pub ttft_p90: Option, + /// Mean streaming time per output token, seconds. + pub tpot: Option, + /// Estimated prefill seconds per uncached input token. + pub prefill_per_token: Option, +} + /// A validated route candidate ready for runtime matching. /// /// Created by `validate_candidates` from raw config entries. All string /// fields are bounded and non-blank. The Grid-owned fields (`admission_state`, /// `rank`, `selection_tier`) are populated after validation from live signals. /// -/// `Clone` is cheap: every owned field is an `Arc`. The refresh step clones +/// `Clone` is cheap: every owned field is reference counted. The refresh step clones /// the base set each poll cycle to re-order it by live load. #[derive(Clone, Debug)] pub struct RouteCandidate { /// Grid-operator admission state. pub admission_state: AdmissionState, + /// Requests this candidate runs at once without engine queueing, as its operator publishes + /// it; `None` when unpublished, which leaves its load unknown. + pub capacity: Option, + + /// Requests held over capacity, as its operator publishes it; `None` when unpublished or + /// stale. Resolved when the snapshot is built, never per request. + pub rho: Option, + + /// Recent latency its operator publishes, each `None` when unpublished or stale. + pub latency: Latency, + /// Cluster name to select. pub cluster: Arc, @@ -152,6 +183,9 @@ pub struct RouteCandidate { /// Deterministic identifier for session affinity binding. pub stable_id: Arc, + + /// Remote site's gateway, dialed directly instead of a `load_balancer` cluster. + pub upstream: Option, } /// Build a deterministic stable ID from candidate identity fields. @@ -200,7 +234,10 @@ pub(crate) fn validate_candidates(raw: Vec) -> Result) -> Result CandidateConfig { let kind: CapabilityKind = serde_yaml::from_str(&format!("\"{kind_str}\"")).unwrap(); CandidateConfig { + admission: AdmissionState::default(), cluster: cluster.to_owned(), credential: None, fresh: true, @@ -384,4 +423,20 @@ mod tests { let err = validate_local_site("").expect_err("should fail"); assert!(err.to_string().contains("local_site must be"), "{err}"); } + + #[test] + fn admission_parses_as_the_operator_writes_it_and_defaults_to_admitted() { + let parse = |json: &str| serde_yaml::from_str::(json).map(|c| c.admission); + let base = r#""kind":"inference_model","name":"m","site":"s","cluster":"c""#; + assert_eq!(parse(&format!("{{{base}}}")).unwrap(), AdmissionState::NewAndExisting); + assert_eq!( + parse(&format!(r#"{{{base},"admission":"none"}}"#)).unwrap(), + AdmissionState::Excluded + ); + assert_eq!( + parse(&format!(r#"{{{base},"admission":"existing_only"}}"#)).unwrap(), + AdmissionState::ExistingOnly + ); + parse(&format!(r#"{{{base},"admission":"maybe"}}"#)).expect_err("an unknown admission is refused"); + } } diff --git a/gateway/ai-grid-filters/src/flow.rs b/gateway/ai-grid-filters/src/flow.rs index 5b7f1373..f238d3e5 100644 --- a/gateway/ai-grid-filters/src/flow.rs +++ b/gateway/ai-grid-filters/src/flow.rs @@ -2,7 +2,7 @@ //! //! The pieces have unit tests in their own crates. These stitch the real //! `PeerScraper`, the `LoadStore`, and `RouteSnapshot::from_store` plus -//! `select_admitted` into one path so the composition is exercised, including +//! `select_spread` into one path so the composition is exercised, including //! the failure cases where a peer is unreachable or untrusted and the router //! must still produce a sound decision. @@ -33,7 +33,7 @@ use tokio_rustls::TlsAcceptor; use crate::{ descriptor::{CandidateConfig, CapabilityKind, RouteCandidate, validate_candidates}, - route::select_admitted, + route::{KeepAll, select_spread}, snapshot::{LOAD_METRIC, RouteSnapshot}, }; @@ -169,6 +169,7 @@ fn candidates(model: &str, sites: &[(&str, &str)]) -> Vec { let raw = sites .iter() .map(|(site, cluster)| CandidateConfig { + admission: crate::descriptor::AdmissionState::default(), cluster: (*cluster).to_owned(), credential: None, fresh: true, @@ -216,7 +217,16 @@ async fn scrape_store_route_picks_the_least_loaded_site() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-b", "the idle site wins end to end"); } @@ -258,7 +268,16 @@ async fn the_scrape_hop_is_mutually_authenticated_then_routes() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-a"); } @@ -314,7 +333,16 @@ async fn loss_of_signal_sorts_the_unmeasured_site_last() { ); // Even though east is busy (90), west has no signal at all, so it sorts // last and the measured site is chosen. Loss of signal is "least preferred". - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!( &*chosen.cluster, "pool-a", "a measured busy site beats an unmeasured one" @@ -348,7 +376,16 @@ async fn an_unreachable_peer_leaves_no_reading_and_the_reachable_one_routes() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-b", "the reachable site is chosen"); } @@ -385,7 +422,16 @@ async fn an_untrusted_peer_is_refused_and_contributes_no_reading() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-b", "only the trusted, measured peer routes"); } diff --git a/gateway/ai-grid-filters/src/health.rs b/gateway/ai-grid-filters/src/health.rs new file mode 100644 index 00000000..5ba8df73 --- /dev/null +++ b/gateway/ai-grid-filters/src/health.rs @@ -0,0 +1,138 @@ +//! Which backend clusters have no healthy endpoint, resolved off the request path. +//! +//! The route filter publishes Praxis's health registry here; the control step reads +//! it on its own tick, so a request never judges raw health. + +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, +}; + +use arc_swap::{ArcSwap, ArcSwapOption}; +use praxis_core::health::{ClusterHealthState, HealthRegistry}; + +/// Clusters whose every endpoint is down, kept by the control step. +#[derive(Debug, Default)] +pub struct ClusterHealth { + /// The pipeline's registry, published by the route filter. + registry: ArcSwapOption, ClusterHealthState>>, + /// The clusters with no healthy endpoint as of the last tick. + down: ArcSwap>>, +} + +impl ClusterHealth { + /// Publish `registry` when it is not the one already held: one lock-free load per request. + pub(crate) fn observe(&self, registry: Option<&HealthRegistry>) { + let Some(registry) = registry else { + return; + }; + let held = self.registry.load(); + if !held.as_ref().is_some_and(|held| Arc::ptr_eq(held, registry)) { + self.registry.store(Some(Arc::clone(registry))); + } + } + + /// Whether praxis has reported any cluster health yet. + /// + /// Without a registry the gateway knows nothing about backend health, so demotion + /// stays off rather than treating every cluster as healthy or as down. + pub(crate) fn observed(&self) -> bool { + self.registry.load().is_some() + } + + /// The clusters that were down as of the last [`Self::update`]. + pub(crate) fn down(&self) -> Arc>> { + self.down.load_full() + } + + /// Recompute the down set from the registry, returning whether it changed. + pub(crate) fn update(&self) -> bool { + let now = self.registry.load().as_deref().map(down_clusters).unwrap_or_default(); + if **self.down.load() == now { + return false; + } + tracing::info!(down = ?now, "grid: backend clusters with no healthy endpoint changed"); + self.down.store(Arc::new(now)); + true + } +} + +/// Clusters in `registry` with endpoints, none of them healthy. +fn down_clusters(registry: &HashMap, ClusterHealthState>) -> BTreeSet> { + registry + .iter() + .filter(|(_, state)| { + let (healthy, total) = state.endpoint_counts(); + total > 0 && healthy == 0 + }) + .map(|(name, _)| Arc::clone(name)) + .collect() +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing, reason = "tests")] +mod tests { + use praxis_core::health::{ClusterHealthEntry, EndpointHealth}; + + use super::*; + + fn registry(clusters: &[(&str, &[bool])]) -> HealthRegistry { + Arc::new( + clusters + .iter() + .map(|(name, endpoints)| { + let health: Vec = endpoints + .iter() + .map(|healthy| { + let endpoint = EndpointHealth::new(); + if !healthy { + endpoint.mark_unhealthy(); + } + endpoint + }) + .collect(); + let addresses = (0..endpoints.len()) + .map(|i| Arc::from(format!("10.0.0.{i}:80"))) + .collect(); + let entry = ClusterHealthEntry::new(health, addresses, None, None); + (Arc::from(*name), Arc::new(entry)) + }) + .collect(), + ) + } + + #[test] + fn a_cluster_is_down_only_when_every_endpoint_is() { + let health = ClusterHealth::default(); + health.observe(Some(®istry(&[ + ("site-a", &[false]), + ("site-b", &[false, true]), + ("site-d", &[true]), + ]))); + assert!(health.update(), "the first tick finds site-a down"); + assert_eq!(*health.down(), BTreeSet::from([Arc::from("site-a")])); + assert!(!health.update(), "an unchanged registry is not news"); + } + + #[test] + fn a_recovered_cluster_leaves_the_down_set() { + let health = ClusterHealth::default(); + health.observe(Some(®istry(&[("site-a", &[false])]))); + assert!(health.update()); + health.observe(Some(®istry(&[("site-a", &[true])]))); + assert!(health.update(), "the recovered cluster leaves the down set"); + assert!(health.down().is_empty()); + } + + #[test] + fn nothing_is_down_and_nothing_is_observed_without_a_registry() { + let health = ClusterHealth::default(); + health.observe(None); + assert!(!health.observed(), "no registry, so demotion stays off"); + assert!(!health.update()); + assert!(health.down().is_empty()); + health.observe(Some(®istry(&[("site-a", &[true])]))); + assert!(health.observed(), "praxis reported, so demotion applies"); + } +} diff --git a/gateway/ai-grid-filters/src/lib.rs b/gateway/ai-grid-filters/src/lib.rs index e723b5f5..458692e5 100644 --- a/gateway/ai-grid-filters/src/lib.rs +++ b/gateway/ai-grid-filters/src/lib.rs @@ -6,9 +6,11 @@ //! contribution is ordering the candidates by live load off the request path. mod control; +mod decisions; mod descriptor; #[cfg(test)] mod flow; +mod health; mod metadata; mod route; mod serving; @@ -18,10 +20,12 @@ use std::sync::Arc; use arc_swap::ArcSwap; pub use control::ReloadOutcome; +pub use decisions::SiteDecisions; // The routing model and the snapshot builder are the crate's control-plane API: // the gateway's refresh step orders candidates by live load and swaps the // snapshot. The request path only reads a snapshot. -pub use descriptor::{AdmissionState, CandidateConfig, CapabilityKind, RouteCandidate}; +pub use descriptor::{AdmissionState, CandidateConfig, CapabilityKind, Latency, RouteCandidate}; +pub use health::ClusterHealth; pub use metadata::{CandidateCredential, CredentialRef}; use praxis_filter::{FilterError, FilterFactory, FilterRegistry, HttpFilter}; pub use serving::{GridRuntime, GridServingConfig, PeerServingConfig, load_serving_config, spawn_grid_routing}; @@ -31,7 +35,7 @@ pub use snapshot::RouteSnapshot; /// owns and its refresh loop swaps. /// /// Call this from the gateway after `FilterRegistry::with_builtins()`, passing -/// the snapshot from [`spawn_grid_routing`]. The factory captures the snapshot, +/// the snapshot and cluster health from [`spawn_grid_routing`]. The factory captures the snapshot, /// so every filter praxis rebuilds on a config reload clones the same `Arc` and /// sees the live swaps. /// @@ -41,9 +45,10 @@ pub use snapshot::RouteSnapshot; pub fn register_grid_filters( registry: &mut FilterRegistry, snapshot: Arc>, + health: Arc, ) -> Result<(), FilterError> { let factory = move |config: &serde_yaml::Value| -> Result, FilterError> { - route::GridSiteRouteFilter::from_config(config, Arc::clone(&snapshot)) + route::GridSiteRouteFilter::from_config(config, Arc::clone(&snapshot), Arc::clone(&health)) }; registry.register("grid_site_route", FilterFactory::Http(Arc::new(factory))) } diff --git a/gateway/ai-grid-filters/src/route.rs b/gateway/ai-grid-filters/src/route.rs index 6b18e852..54e1822f 100644 --- a/gateway/ai-grid-filters/src/route.rs +++ b/gateway/ai-grid-filters/src/route.rs @@ -1,19 +1,31 @@ //! The `grid_site_route` filter: pick a cross-site cluster for a request. //! -//! Reads the model header, finds the front admitted candidate for that model in -//! the current snapshot, and sets `ctx.cluster` for the downstream load -//! balancer. Selection is `select_admitted` over a pre-ordered list. The -//! ordering by live load happens off the request path in `snapshot`. +//! Reads the model header, chooses a site for that model from the current snapshot +//! by what each provider publishes, and sets `ctx.cluster` for the downstream load +//! balancer. Each site's load, requests held over capacity (rho), is resolved when +//! the snapshot is built, so a request only filters and draws. A request naming no +//! model gets 400, one for an unknown model 404, and one with no healthy site or a +//! shed model 503 with Retry-After and an OpenAI-style error, logged at debug. Every outcome counts in +//! `grid_route_decisions_total` by site and reason. A routed response names the +//! chosen site and cluster in `x-grid-site` and `x-grid-backend`. -use std::sync::Arc; +use std::{ + collections::HashSet, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; use arc_swap::ArcSwap; use async_trait::async_trait; -use praxis_filter::{FilterAction, FilterError, HttpFilter, HttpFilterContext, Rejection, parse_filter_config}; +use praxis_filter::{FilterAction, FilterError, HttpFilter, HttpFilterContext, TerminalResponse, parse_filter_config}; use serde::Deserialize; use crate::{ + decisions::{Refused, SiteDecisions}, descriptor::{AdmissionState, CapabilityKind, RouteCandidate, validate_model_header}, + health::ClusterHealth, snapshot::RouteSnapshot, }; @@ -33,6 +45,90 @@ struct GridSiteRouteConfig { /// Request header naming the model (default `X-Model`). #[serde(default = "default_model_header")] model_header: String, + + /// The `load_balancer` clusters this gateway declares. When set, a candidate + /// with no peer gateway and a cluster not listed has no route and is skipped. + #[serde(default)] + clusters: Option>, +} + +/// How a cluster's endpoints are dialed, as its `load_balancer` cluster declares. +/// +/// Parsed and validated but unused: nothing probes endpoints in this filter yet, and a +/// config carrying a transport must still load rather than fail the gateway's start. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +enum TransportKind { + /// Grid mutual TLS with the site identity. + MutualTls, + /// Server-only TLS, verified with the cluster's CA. + Tls, + /// No TLS. + Plaintext, +} + +/// One declared cluster: a bare name, or a name with the transport its endpoints are dialed over. +#[derive(Debug)] +enum DeclaredCluster { + /// Routable, not probed. + Name(String), + /// Routable, and carrying the transport its endpoints are dialed over. + Probed(ProbedCluster), +} + +// By hand, not untagged: untagged hides which field or transport was wrong. +impl<'de> Deserialize<'de> for DeclaredCluster { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = DeclaredCluster; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a cluster name or {name, transport}") + } + + fn visit_str(self, v: &str) -> Result { + Ok(DeclaredCluster::Name(v.to_owned())) + } + + fn visit_map>(self, map: A) -> Result { + ProbedCluster::deserialize(serde::de::value::MapAccessDeserializer::new(map)) + .map(DeclaredCluster::Probed) + } + } + + deserializer.deserialize_any(Visitor) + } +} + +/// A declared cluster that names how its remote endpoints are dialed. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[expect( + dead_code, + reason = "parsed so a config carrying a transport loads; nothing probes yet" +)] +struct ProbedCluster { + /// The `load_balancer` cluster name. + name: String, + /// How the cluster dials its endpoints. + transport: TransportKind, + /// The CA a `tls` cluster verifies with. + #[serde(default)] + ca_path: Option, + /// The server name a `tls` cluster expects. + #[serde(default)] + sni: Option, +} + +impl DeclaredCluster { + /// The cluster name. + fn name(&self) -> &str { + match self { + Self::Name(name) | Self::Probed(ProbedCluster { name, .. }) => name, + } + } } /// Routes a request to a cross-site cluster by model, honouring live-load order. @@ -44,6 +140,15 @@ pub(crate) struct GridSiteRouteFilter { /// Header the request carries the model name in. model_header: http::header::HeaderName, + + /// Counts requests from a random start, so gateway replicas draw independently. + turn: AtomicUsize, + + /// Where the filter publishes Praxis's health registry for the control step. + health: Arc, + + /// Declared clusters, `None` when the config does not list them. + clusters: Option>>, } impl GridSiteRouteFilter { @@ -58,10 +163,40 @@ impl GridSiteRouteFilter { pub(crate) fn from_config( config: &serde_yaml::Value, snapshot: Arc>, + health: Arc, ) -> Result, FilterError> { let cfg: GridSiteRouteConfig = parse_filter_config("grid_site_route", config)?; let model_header = validate_model_header(&cfg.model_header)?; - Ok(Box::new(Self { snapshot, model_header })) + let declared = cfg.clusters; + tracing::info!( + "grid_site_route: choosing sites by published in-flight per capacity: two choices among 3 or more \ + sites with room, a weighted draw between 2, and a capacity draw when none has room or none publishes" + ); + Ok(Box::new(Self { + snapshot, + model_header, + turn: AtomicUsize::new(random_seed()), + health, + clusters: declared.map(|clusters| clusters.iter().map(|cluster| Arc::from(cluster.name())).collect()), + })) + } + + /// The site for a request for `model`, or the answer when no site can take it. + fn choose<'snap>(&self, snapshot: &'snap RouteSnapshot, model: &str) -> Result, FilterAction> { + let turn = self.turn.fetch_add(1, Ordering::Relaxed); + if snapshot.shedding.contains(model) { + return Err(refuse(Refused::Shed, model, turn)); + } + let routable = |candidate: &RouteCandidate| has_route(candidate, self.clusters.as_ref()); + select_spread( + snapshot, + CapabilityKind::InferenceModel, + model, + turn, + routable, + &KeepAll, + ) + .ok_or_else(|| refuse(unserved(snapshot, model, routable), model, turn)) } } @@ -76,8 +211,9 @@ impl HttpFilter for GridSiteRouteFilter { } async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { - // An earlier cluster-selecting filter wins; never override its choice. - if ctx.cluster.is_some() { + self.health.observe(ctx.health_registry); + // An earlier selecting filter wins; never override its choice. + if ctx.cluster.is_some() || ctx.upstream.is_some() { return Ok(FilterAction::Continue); } // Borrow the model header out of *ctx.request; that field is disjoint @@ -88,35 +224,416 @@ impl HttpFilter for GridSiteRouteFilter { .get(&self.model_header) .and_then(|value| value.to_str().ok()) else { - // No model header: not ours to route, leave it for the next filter. - return Ok(FilterAction::Continue); + // Nothing after this filter selects a cluster, so answer here rather than fail in the load balancer. + tracing::debug!(path = %ctx.request.uri.path(), "grid_site_route: no model in the request"); + Refused::NoModel.record(); + return Ok(unrouted(400)); }; let snapshot = self.snapshot.load(); - let Some(candidate) = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, model) else { - tracing::debug!(model = %model, "grid_site_route: no admitted candidate"); - return Ok(FilterAction::Reject(Rejection::status(404))); + match self.choose(&snapshot, model) { + Ok(pick) => { + route_to(ctx, &pick); + Ok(FilterAction::Continue) + }, + Err(answer) => Ok(answer), + } + } +} + +/// Send the request to `pick`: count the decision and select its cluster. +fn route_to(ctx: &mut HttpFilterContext<'_>, pick: &Pick<'_>) { + pick.decisions.record(pick.fallback); + ctx.cluster = Some(Arc::clone(&pick.candidate.cluster)); +} + +/// Count and answer a request no candidate took. +fn refuse(reason: Refused, model: &str, turn: usize) -> FilterAction { + tracing::debug!(model = %model, reason = ?reason, "grid_site_route: no admitted candidate"); + reason.record(); + // Known but excluded everywhere is a temporary outage, not an unknown model. + match reason { + Refused::UnknownModel => error_response( + 404, + "invalid_request_error", + "model_not_found", + "no site serves this model", + None, + ), + // Load, not an outage: 429 is what an OpenAI client backs off and retries on. + Refused::Shed => retry_later( + turn, + 429, + "rate_limit_exceeded", + "capacity_exhausted", + "every site serving this model is at capacity", + ), + Refused::NotReady | Refused::NoRoute | Refused::NoModel => retry_later( + turn, + 503, + "server_error", + "no_healthy_site", + "no healthy site serves this model now", + ), + } +} + +/// Why no candidate took a request for `model`: unknown, unroutable, or excluded. +fn unserved(snapshot: &RouteSnapshot, model: &str, routable: impl Fn(&RouteCandidate) -> bool) -> Refused { + let mut matches = snapshot + .candidates + .iter() + .filter(|candidate| candidate.kind == CapabilityKind::InferenceModel && &*candidate.name == model) + .peekable(); + if matches.peek().is_none() { + return Refused::UnknownModel; + } + if matches.any(|candidate| is_admitted_for_new_request(candidate.admission_state) && !routable(candidate)) { + Refused::NoRoute + } else { + Refused::NotReady + } +} + +/// The candidate `select_spread` chose and why. +#[derive(Debug)] +pub(crate) struct Pick<'snap> { + /// The chosen candidate. + pub(crate) candidate: &'snap RouteCandidate, + /// Whether it was demoted, chosen only because no healthy site was left. + pub(crate) fallback: bool, + /// The chosen site's decision counters. + pub(crate) decisions: &'snap SiteDecisions, +} + +impl<'snap> Pick<'snap> { + /// `site` as chosen, a `fallback` when no healthy site was left. + const fn new((candidate, decisions): Site<'snap>, fallback: bool) -> Self { + Self { + candidate, + fallback, + decisions, + } + } +} + +/// The router's own answer to a request it cannot route: a response, not a rejection, +/// since praxis logs every rejection at WARN. +fn unrouted(status: u16) -> FilterAction { + FilterAction::TerminalResponse(Box::new(TerminalResponse::new(status))) +} + +/// Seconds a client should wait before retrying a model with no admitted candidate: +/// about one operator scrape plus one peer poll, spread so retries do not arrive together. +const RETRY_AFTER_SECS: [&str; 5] = ["3", "4", "5", "6", "7"]; + +/// `status` with a Retry-After picked by `turn` and an OpenAI-style error naming `code`. +fn retry_later(turn: usize, status: u16, kind: &str, code: &str, message: &str) -> FilterAction { + let retry_after = turn + .checked_rem(RETRY_AFTER_SECS.len()) + .and_then(|index| RETRY_AFTER_SECS.get(index)) + .copied() + .unwrap_or("5"); + error_response(status, kind, code, message, Some(retry_after)) +} + +/// A gateway-originated error: `status`, an OpenAI-style error object, and an optional Retry-After. +fn error_response( + status: u16, + kind: &str, + code: &str, + message: &str, + retry_after: Option<&'static str>, +) -> FilterAction { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + if let Some(retry_after) = retry_after { + headers.insert(http::header::RETRY_AFTER, http::HeaderValue::from_static(retry_after)); + } + // Every code and message is a constant here, so none needs JSON escaping. + let body = format!(r#"{{"error":{{"message":"{message}","type":"{kind}","code":"{code}"}}}}"#); + FilterAction::TerminalResponse(Box::new( + TerminalResponse::new(status).with_headers(headers).with_body(body), + )) +} + +/// Whether a request routed to `candidate` has somewhere to go: a declared cluster. +/// With no declared list, every cluster is assumed to exist. +fn has_route(candidate: &RouteCandidate, clusters: Option<&HashSet>>) -> bool { + clusters.is_none_or(|clusters| clusters.contains(&candidate.cluster)) +} + +/// One site the draw may choose, as a [`Narrow`] stage sees it. +#[derive(Clone, Copy, Debug)] +#[cfg_attr( + not(test), + expect(dead_code, reason = "read by narrowing stages; this crate ships only KeepAll") +)] +pub(crate) struct SiteView<'snap> { + /// The candidate. + pub(crate) candidate: &'snap RouteCandidate, + /// Requests it holds over its capacity, below 1 for every site the draw sees. + pub(crate) rho: f64, + /// Expected delay in seconds; `None` until ranking uses latency. + pub(crate) delay: Option, + /// Recent latency its operator publishes, each `None` until enough requests complete. + pub(crate) latency: crate::descriptor::Latency, +} + +/// A per-request stage that narrows the sites with room before the draw. +/// +/// It may only clear entries of `keep`, which start all true, so it never adds a site +/// that health, readiness, or shedding removed. Clearing every entry keeps them all. +pub(crate) trait Narrow { + /// Clear `keep[i]` for each of `sites` the draw should not consider. + fn keep(&self, sites: &[SiteView<'_>], keep: &mut [bool]); +} + +/// Keeps every site: selection by load alone. +pub(crate) struct KeepAll; + +impl Narrow for KeepAll { + fn keep(&self, _sites: &[SiteView<'_>], _keep: &mut [bool]) {} +} + +/// Most sites with room one request weighs. Past it, the rest wait for a later snapshot order. +const MAX_SITES: usize = 64; + +/// Bounds of either site's share when two sites with room are drawn between. +const TWO_SITE_SHARE: (f64, f64) = (0.1, 0.9); + +/// A candidate with its decision counters. +type Site<'snap> = (&'snap RouteCandidate, &'snap SiteDecisions); + +/// Up to [`MAX_SITES`] sites, the first `len` filled. +struct Sites<'snap> { + /// The sites, filled from the front. + slots: [Option>; MAX_SITES], + /// How many slots are filled. + len: usize, +} + +impl<'snap> Sites<'snap> { + /// The first [`MAX_SITES`] of `sites`, warning once when more are offered. + fn gather(sites: impl Iterator>) -> Self { + let mut gathered = Self { + slots: [None; MAX_SITES], + len: 0, }; - ctx.cluster = Some(Arc::clone(&candidate.cluster)); - Ok(FilterAction::Continue) + for site in sites { + let Some(slot) = gathered.slots.get_mut(gathered.len) else { + warn_past_max_sites(); + break; + }; + *slot = Some(site); + gathered.len = gathered.len.saturating_add(1); + } + gathered + } + + /// The filled sites in order. + fn iter(&self) -> impl Iterator> + '_ { + self.slots.iter().take(self.len).flatten().copied() + } + + /// The site at `index`, if filled. + fn get(&self, index: usize) -> Option> { + self.slots.get(..self.len)?.get(index).copied().flatten() } } -/// The front candidate matching `kind` and `name` that admits new requests. +/// Choose a site for `name` among the admitted, `routable` matches. /// -/// The list is pre-ordered (least-loaded first), so the front match is the -/// chosen target. This is the reused selection: a linear first-match, never a -/// score computed here. -pub(crate) fn select_admitted<'list>( - candidates: &'list [RouteCandidate], +/// Among healthy matches with room (rho below 1) that `narrow` keeps: two choices at +/// three or more, drawn by capacity, taking the lower rho; a draw between two, weighted +/// by capacity over 1 + rho with each share clamped to 0.1..0.9; or the only one. With +/// none, a capacity draw over the healthy matches tied on the best polled queue score (a +/// prefix, since scores ascend), which covers sites that publish no rho without herding. With no healthy +/// match, the front demoted one, as a fallback. +#[expect( + clippy::too_many_arguments, + reason = "the request path passes its inputs directly rather than build a struct per request" +)] +pub(crate) fn select_spread<'snap>( + snapshot: &'snap RouteSnapshot, kind: CapabilityKind, name: &str, -) -> Option<&'list RouteCandidate> { - candidates.iter().find(|candidate| { - candidate.kind == kind && &*candidate.name == name && is_admitted_for_new_request(candidate.admission_state) + turn: usize, + routable: impl Fn(&RouteCandidate) -> bool, + narrow: &impl Narrow, +) -> Option> { + let matches = matching(snapshot, kind, name, &routable); + let (front, _, front_decisions) = matches.clone().next()?; + let healthy = matches.filter(|(_, score, _)| !score.is_nan()); + let Some((_, best, _)) = healthy.clone().next() else { + // Demotion scores a candidate NaN: every healthy site is gone, so try the front one. + return Some(Pick::new((front, front_decisions), true)); + }; + let site = |(candidate, _, decisions): Scored<'snap>| (candidate, decisions); + let room = Sites::gather(healthy.clone().filter(|(candidate, ..)| has_room(candidate)).map(site)); + let band = || { + Sites::gather( + healthy + .take_while(|(_, score, _)| score.total_cmp(&best).is_eq()) + .map(site), + ) + }; + draw(&narrowed(&room, narrow), band, turn).map(|chosen| Pick::new(chosen, false)) +} + +/// A candidate with its score and decision counters. +type Scored<'snap> = (&'snap RouteCandidate, f64, &'snap SiteDecisions); + +/// The admitted, `routable` candidates for `kind` and `name`, in snapshot order. +fn matching<'snap, 'req>( + snapshot: &'snap RouteSnapshot, + kind: CapabilityKind, + name: &'req str, + routable: &'req impl Fn(&RouteCandidate) -> bool, +) -> impl Iterator> + Clone + 'req +where + 'snap: 'req, +{ + snapshot + .candidates + .iter() + .zip(&snapshot.scores) + .zip(&snapshot.decisions) + .map(|((candidate, score), decisions)| (candidate, *score, decisions)) + .filter(move |(candidate, ..)| { + candidate.kind == kind + && &*candidate.name == name + && is_admitted_for_new_request(candidate.admission_state) + && routable(candidate) + }) +} + +/// The tier for `room` sites: two choices at three or more, a weighted draw at two, the +/// only one at one, and with none a capacity draw over `band`. +fn draw<'snap>(room: &Sites<'snap>, band: impl FnOnce() -> Sites<'snap>, turn: usize) -> Option> { + match room.len { + 0 => by_capacity(&band(), turn), + 1 => room.get(0), + 2 => Some(between_two(room.get(0)?, room.get(1)?, turn)), + _ => two_choices(room, turn), + } +} + +/// Whether `candidate` publishes load and has room for one more request. +fn has_room(candidate: &RouteCandidate) -> bool { + candidate.rho.is_some_and(|rho| rho < 1.0) +} + +/// The sites `narrow` keeps of `room`, or all of them when it keeps none. +fn narrowed<'snap>(room: &Sites<'snap>, narrow: &impl Narrow) -> Sites<'snap> { + let views: [Option>; MAX_SITES] = std::array::from_fn(|index| { + room.get(index).map(|(candidate, _)| SiteView { + candidate, + rho: candidate.rho.unwrap_or(0.0), + delay: None, + latency: candidate.latency, + }) + }); + let views: Vec> = views.iter().flatten().copied().collect(); + let mut keep = [true; MAX_SITES]; + if let Some(kept) = keep.get_mut(..room.len) { + narrow.keep(&views, kept); + } + if !keep.iter().take(room.len).any(|kept| *kept) { + return Sites::gather(room.iter()); + } + Sites::gather(room.iter().zip(keep).filter(|(_, kept)| *kept).map(|(site, _)| site)) +} + +/// A capacity draw over `sites`. Unpublished capacity weighs the mean of the published, or 1. +fn by_capacity<'snap>(sites: &Sites<'snap>, turn: usize) -> Option> { + let published: Vec = sites.iter().filter_map(|(candidate, _)| candidate.capacity).collect(); + let unpublished = if published.is_empty() { + 1.0 + } else { + published.iter().sum::() / published.iter().map(|_| 1.0).sum::() + }; + weighted(sites, unit(turn, SALTS.0), None, |candidate| { + candidate.capacity.unwrap_or(unpublished) }) } +/// A draw between two sites by capacity over 1 + rho, each share clamped to 0.1..0.9. +fn between_two<'snap>(first: Site<'snap>, second: Site<'snap>, turn: usize) -> Site<'snap> { + let rate = |candidate: &RouteCandidate| candidate.capacity.unwrap_or(1.0) / (1.0 + candidate.rho.unwrap_or(0.0)); + let share = (rate(first.0) / (rate(first.0) + rate(second.0))).clamp(TWO_SITE_SHARE.0, TWO_SITE_SHARE.1); + if unit(turn, SALTS.0) < share { first } else { second } +} + +/// Two distinct draws from `sites` by capacity, taking the lower rho; exact ties go to the first. +fn two_choices<'snap>(sites: &Sites<'snap>, turn: usize) -> Option> { + let capacity = |candidate: &RouteCandidate| candidate.capacity.unwrap_or(1.0); + let first = weighted(sites, unit(turn, SALTS.0), None, capacity)?; + let second = weighted(sites, unit(turn, SALTS.1), Some(first.0), capacity).unwrap_or(first); + Some(if second.0.rho < first.0.rho { second } else { first }) +} + +/// The site `draw` (in 0..1) lands on when `sites` are weighted by `weight`, skipping `skip`. +fn weighted<'snap>( + sites: &Sites<'snap>, + draw: f64, + skip: Option<&RouteCandidate>, + weight: impl Fn(&RouteCandidate) -> f64, +) -> Option> { + let eligible = || { + sites + .iter() + .filter(|(candidate, _)| skip.is_none_or(|skip| !std::ptr::eq(*candidate, skip))) + }; + let mut target = draw * eligible().map(|(candidate, _)| weight(candidate)).sum::(); + let mut last = None; + for site in eligible() { + last = Some(site); + target -= weight(site.0); + if target < 0.0 { + return Some(site); + } + } + last +} + +/// Salts that make a request's two draws independent. +const SALTS: (u64, u64) = (0x9E37_79B9_7F4A_7C15, 0xC2B2_AE3D_27D4_EB4F); + +/// A uniform value in [0, 1) from `turn` and `salt`, mixed so consecutive turns do not correlate. +fn unit(turn: usize, salt: u64) -> f64 { + let mut mixed = u64::try_from(turn).unwrap_or(u64::MAX) ^ salt; + mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + mixed ^= mixed >> 31; + // The top 53 bits as a fraction, built from two exact halves. + let bits = mixed >> 11; + let high = u32::try_from(bits >> 21).unwrap_or(0); + let low = u32::try_from(bits & 0x1F_FFFF).unwrap_or(0); + (f64::from(high) * 2_f64.powi(21) + f64::from(low)) / 2_f64.powi(53) +} + +/// A random starting turn, so replicas started together do not draw in step. +fn random_seed() -> usize { + use std::hash::BuildHasher as _; + let hashed = std::collections::hash_map::RandomState::new().hash_one(std::time::SystemTime::now()); + usize::try_from(hashed).unwrap_or_else(|_| usize::try_from(hashed >> 32).unwrap_or(0)) +} + +/// Log once that a model has more sites than one request weighs. +fn warn_past_max_sites() { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + tracing::warn!( + max = MAX_SITES, + "grid_site_route: more sites than one request weighs; the rest wait for a later order" + ); + }); +} + /// Whether a candidate in this admission state accepts a new request. fn is_admitted_for_new_request(state: AdmissionState) -> bool { matches!(state, AdmissionState::NewAndExisting) @@ -130,15 +647,25 @@ fn is_admitted_for_new_request(state: AdmissionState) -> bool { clippy::indexing_slicing, clippy::panic, clippy::min_ident_chars, + clippy::type_complexity, + clippy::arithmetic_side_effects, + clippy::float_arithmetic, reason = "tests" )] mod tests { use super::*; use crate::descriptor::{CandidateConfig, validate_candidates}; + /// The pick for `kind` and `name` over `candidates` in their given order. + fn front(candidates: Vec, kind: CapabilityKind, name: &str) -> Option { + let snapshot = RouteSnapshot::from_static(candidates, Arc::from("east")); + select_spread(&snapshot, kind, name, 0, |_| true, &KeepAll).map(|pick| pick.candidate.cluster.to_string()) + } + /// A validated one-candidate list for `model` at `site`/`cluster`. fn one(model: &str, site: &str, cluster: &str, admission: AdmissionState) -> Vec { let mut candidates = validate_candidates(vec![CandidateConfig { + admission: AdmissionState::default(), cluster: cluster.to_owned(), credential: None, fresh: true, @@ -151,28 +678,417 @@ mod tests { candidates } + /// Three tied candidates for `m`, at sites a, b and d. + fn three() -> Vec { + ["a", "b", "d"] + .iter() + .flat_map(|site| one("m", site, &format!("pool-{site}"), AdmissionState::NewAndExisting)) + .collect() + } + + fn pick(snapshot: &RouteSnapshot, turn: usize) -> String { + select_spread(snapshot, CapabilityKind::InferenceModel, "m", turn, |_| true, &KeepAll) + .map(|chosen| chosen.candidate.cluster.to_string()) + .unwrap() + } + + #[test] + fn tied_candidates_split_evenly() { + let snapshot = RouteSnapshot::from_static(three(), Arc::from("hub")); + let picks: Vec = (0..3_000).map(|turn| pick(&snapshot, turn)).collect(); + for cluster in ["pool-a", "pool-b", "pool-d"] { + let taken = picks.iter().filter(|p| *p == cluster).count(); + assert!((850..=1_150).contains(&taken), "{cluster} took {taken} of 3000"); + } + } + + #[test] + fn a_declared_cluster_is_a_name_or_a_name_with_its_transport() { + let cfg: GridSiteRouteConfig = serde_yaml::from_str( + "clusters:\n - site-a\n - name: east\n transport: tls\n ca_path: /etc/ca.crt\n sni: east.svc\n - name: west\n transport: plaintext\n", + ) + .expect("a config carrying transports still loads"); + let declared = cfg.clusters.expect("declared"); + let names: Vec<&str> = declared.iter().map(DeclaredCluster::name).collect(); + assert_eq!(names, ["site-a", "east", "west"], "every declared cluster is routable"); + let unknown = serde_yaml::from_str::("clusters:\n - name: east\n transport: quic\n") + .expect_err("an unknown transport is rejected"); + assert!(unknown.to_string().contains("quic"), "{unknown}"); + assert!( + serde_yaml::from_str::("clusters:\n - name: east\n transport: tls\n ca: /x\n") + .is_err(), + "an unknown key is rejected" + ); + } + + #[test] + fn a_candidate_with_no_peer_gateway_and_no_declared_cluster_has_no_route() { + let mut candidates = one("llama", "site-d", "site-d", AdmissionState::NewAndExisting); + candidates.extend(one("llama", "dagobah", "site-a", AdmissionState::NewAndExisting)); + let snapshot = RouteSnapshot::from_static(candidates, Arc::from("dagobah")); + let declared: HashSet> = HashSet::from([Arc::from("site-a")]); + for turn in 0..4 { + let picked = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |c| has_route(c, Some(&declared)), + &KeepAll, + ) + .expect("site-a is routable") + .candidate; + assert_eq!( + &*picked.cluster, "site-a", + "site-d is not a declared cluster, turn {turn}" + ); + } + assert!( + snapshot.candidates.iter().all(|c| has_route(c, None)), + "with no declared list, every cluster is assumed to exist" + ); + let undeclared = one("llama", "site-d", "site-d", AdmissionState::NewAndExisting).remove(0); + assert!( + !has_route(&undeclared, Some(&declared)), + "a cluster the config never declared is no route" + ); + } + + /// Whether the pick at `turn` for `llama` over `snapshot` was a fallback. + fn fallback(snapshot: &RouteSnapshot, turn: usize) -> Option { + select_spread( + snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |_| true, + &KeepAll, + ) + .map(|pick| pick.fallback) + } + + #[test] + fn a_pick_is_a_fallback_only_when_no_healthy_site_is_left() { + assert_eq!(fallback(&two_sites(Some(1.0), Some(5.0)), 0), Some(false)); + assert_eq!(fallback(&two_sites(Some(2.0), Some(2.0)), 1), Some(false)); + let alone = RouteSnapshot::from_static( + one("llama", "east", "pool-a", AdmissionState::NewAndExisting), + Arc::from("east"), + ); + assert_eq!(fallback(&alone, 0), Some(false)); + let down = std::collections::BTreeSet::from([Arc::from("pool-a"), Arc::from("pool-b")]); + assert_eq!(fallback(&two_sites(Some(1.0), Some(5.0)).demote(&down), 0), Some(true)); + // One cluster left healthy is a routed pick, not a fallback. + let one = std::collections::BTreeSet::from([Arc::from("pool-a")]); + assert_eq!(fallback(&two_sites(Some(1.0), Some(5.0)).demote(&one), 0), Some(false)); + } + + #[test] + fn an_unserved_request_names_why() { + let mut candidates = one("llama", "east", "pool-a", AdmissionState::Excluded); + candidates.extend(one("llama", "west", "pool-b", AdmissionState::NewAndExisting)); + let snapshot = RouteSnapshot::from_static(candidates, Arc::from("east")); + assert_eq!(unserved(&snapshot, "granite", |_| true), Refused::UnknownModel); + assert_eq!(unserved(&snapshot, "llama", |_| false), Refused::NoRoute); + let excluded = RouteSnapshot::from_static( + one("llama", "east", "pool-a", AdmissionState::Excluded), + Arc::from("east"), + ); + assert_eq!(unserved(&excluded, "llama", |_| false), Refused::NotReady); + } + #[test] fn front_match_is_selected() { let candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); - let chosen = select_admitted(&candidates, CapabilityKind::InferenceModel, "llama").expect("a match"); - assert_eq!(&*chosen.cluster, "pool-a"); + assert_eq!( + front(candidates, CapabilityKind::InferenceModel, "llama").as_deref(), + Some("pool-a") + ); } #[test] fn a_different_model_does_not_match() { let candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); - assert!(select_admitted(&candidates, CapabilityKind::InferenceModel, "granite").is_none()); + assert!(front(candidates, CapabilityKind::InferenceModel, "granite").is_none()); } #[test] fn an_excluded_candidate_is_skipped() { let candidates = one("llama", "east", "pool-a", AdmissionState::Excluded); - assert!(select_admitted(&candidates, CapabilityKind::InferenceModel, "llama").is_none()); + assert!(front(candidates, CapabilityKind::InferenceModel, "llama").is_none()); } #[test] fn an_mcp_kind_does_not_match_an_inference_query() { let candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); - assert!(select_admitted(&candidates, CapabilityKind::McpTool, "llama").is_none()); + assert!(front(candidates, CapabilityKind::McpTool, "llama").is_none()); + } + + /// A snapshot of `llama` on east and west, with the given queue depths. + fn two_sites(east: Option, west: Option) -> RouteSnapshot { + let store = grid_signals::LoadStore::new(std::time::Duration::from_secs(60)); + for (site, cluster, load) in [("east", "pool-a", east), ("west", "pool-b", west)] { + if let Some(value) = load { + let line = format!( + r#"{}{{grid_site="{site}",grid_provider="{cluster}"}} {value} 1000"#, + crate::snapshot::LOAD_METRIC + ); + store.ingest_at(&line, 1_000, 1_000, site); + } + } + let mut candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); + candidates.extend(one("llama", "west", "pool-b", AdmissionState::NewAndExisting)); + RouteSnapshot::from_store(candidates, Arc::from("east"), &store, 1_000, 30_000) + } + + fn picks(snapshot: &RouteSnapshot) -> Vec { + (0..2_000) + .map(|turn| { + let chosen = select_spread( + snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |_| true, + &KeepAll, + ) + .unwrap() + .candidate; + chosen.site.to_string() + }) + .collect() + } + + #[test] + fn equal_scores_spread_across_the_tied_candidates() { + for (snapshot, case) in [ + (two_sites(None, None), "unmeasured"), + (two_sites(Some(5.0), Some(5.0)), "measured"), + ] { + let east = picks(&snapshot).iter().filter(|site| *site == "east").count(); + assert!((850..=1_150).contains(&east), "{case}: east took {east} of 2000"); + } + } + + #[test] + fn a_better_score_always_wins() { + assert_eq!(picks(&two_sites(Some(9.0), Some(1.0))), ["west"; 2_000]); + assert_eq!( + picks(&two_sites(Some(1.0), None)), + ["east"; 2_000], + "measured beats unmeasured" + ); + } + + #[test] + fn an_excluded_tie_is_never_picked() { + let mut snapshot = two_sites(None, None); + snapshot.candidates[1].admission_state = AdmissionState::Excluded; + assert_eq!(picks(&snapshot), ["east"; 2_000]); + } + + /// Sites publishing (in flight, capacity), each `None` to publish nothing, as one snapshot. + fn loaded(sites: &[(&str, Option, Option)]) -> RouteSnapshot { + let store = grid_signals::LoadStore::new(std::time::Duration::from_secs(60)); + let mut candidates = Vec::new(); + for (site, in_flight, capacity) in sites { + let cluster = format!("pool-{site}"); + let labels = format!(r#"grid_site="{site}",grid_provider="{cluster}""#); + let saturation = in_flight.zip(*capacity).map(|(held, capacity)| held / capacity); + for (metric, value) in [ + ("grid_provider_saturation_ratio", saturation), + ("grid_provider_capacity_requests", *capacity), + ] { + if let Some(value) = value { + store.ingest_at(&format!("{metric}{{{labels}}} {value} 1000"), 1_000, 1_000, site); + } + } + candidates.extend(one("llama", site, &cluster, AdmissionState::NewAndExisting)); + } + RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000) + } + + /// How many of `draws` requests each site took. + fn shares( + snapshot: &RouteSnapshot, + draws: usize, + narrow: &impl Narrow, + ) -> std::collections::BTreeMap { + let mut taken = std::collections::BTreeMap::new(); + for turn in 0..draws { + let chosen = select_spread( + snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |_| true, + narrow, + ) + .unwrap(); + *taken.entry(chosen.candidate.site.to_string()).or_insert(0) += 1; + } + taken + } + + #[test] + fn two_choices_prefer_the_lighter_site_and_never_a_full_one() { + let snapshot = loaded(&[ + ("a", Some(2.0), Some(10.0)), + ("b", Some(8.0), Some(10.0)), + ("c", Some(5.0), Some(10.0)), + ("d", Some(10.0), Some(10.0)), + ]); + let taken = shares(&snapshot, 3_000, &KeepAll); + assert_eq!(taken.get("d"), None, "rho 1 has no room"); + let (a, c) = (taken["a"], taken["c"]); + assert!(a > c && c > 0, "the lighter site takes more: a {a}, c {c}"); + assert_eq!( + taken.get("b"), + None, + "two distinct draws never choose the heaviest of three" + ); + } + + #[test] + fn two_sites_split_by_capacity_over_load_within_a_tenth_and_nine_tenths() { + let even = shares( + &loaded(&[("a", Some(1.0), Some(10.0)), ("b", Some(1.0), Some(10.0))]), + 4_000, + &KeepAll, + ); + assert!( + (1_800..=2_200).contains(&even["a"]), + "equal sites split evenly: {even:?}" + ); + let lopsided = shares( + &loaded(&[("a", Some(0.0), Some(100.0)), ("b", Some(9.0), Some(10.0))]), + 4_000, + &KeepAll, + ); + assert!( + (280..=520).contains(&lopsided["b"]), + "the clamp keeps a tenth for the slow site: {lopsided:?}" + ); + } + + #[test] + fn a_lone_site_with_room_takes_every_request() { + let snapshot = loaded(&[("a", Some(12.0), Some(10.0)), ("b", Some(3.0), Some(10.0))]); + assert_eq!(shares(&snapshot, 200, &KeepAll).get("b"), Some(&200)); + } + + #[test] + fn with_no_room_anywhere_the_overflow_draws_by_capacity() { + let snapshot = loaded(&[("a", Some(30.0), Some(30.0)), ("b", Some(10.0), Some(10.0))]); + let taken = shares(&snapshot, 4_000, &KeepAll); + assert!( + (2_700..=3_300).contains(&taken["a"]), + "a has three quarters of the capacity: {taken:?}" + ); + } + + #[test] + fn sites_without_published_capacity_are_drawn_evenly() { + let snapshot = loaded(&[("a", Some(5.0), None), ("b", Some(1.0), None), ("c", None, None)]); + let taken = shares(&snapshot, 3_000, &KeepAll); + for site in ["a", "b", "c"] { + assert!((850..=1_150).contains(&taken[site]), "no rho, no load input: {taken:?}"); + } + } + + #[test] + fn a_demoted_site_is_never_drawn_while_another_is_healthy() { + let down = std::collections::BTreeSet::from([Arc::from("pool-a")]); + let snapshot = loaded(&[ + ("a", Some(0.0), Some(100.0)), + ("b", Some(5.0), Some(10.0)), + ("c", Some(5.0), Some(10.0)), + ]) + .demote(&down); + assert_eq!(shares(&snapshot, 500, &KeepAll).get("a"), None); + } + + /// Keeps only site `b`, or clears everything when `all` is set. + struct Only { + all: bool, + } + + impl Narrow for Only { + fn keep(&self, sites: &[SiteView<'_>], keep: &mut [bool]) { + for (site, kept) in sites.iter().zip(keep) { + *kept = !self.all + && &*site.candidate.site == "b" + && site.rho < 1.0 + && site.delay.is_none() + && site.latency.ttft_p90.is_none(); + } + } + } + + #[test] + fn a_narrowing_stage_can_only_remove_sites_and_clearing_all_keeps_all() { + let snapshot = loaded(&[ + ("a", Some(1.0), Some(10.0)), + ("b", Some(5.0), Some(10.0)), + ("c", Some(12.0), Some(10.0)), + ]); + assert_eq!(shares(&snapshot, 300, &Only { all: false }).get("b"), Some(&300)); + let taken = shares(&snapshot, 300, &Only { all: true }); + assert_eq!(taken.get("c"), None, "a full site is never added back"); + assert!(taken.contains_key("a") && taken.contains_key("b"), "{taken:?}"); + } + + #[test] + fn a_shed_answers_429_and_an_outage_503_each_with_an_openai_error() { + // Load: an OpenAI client backs off on 429 and retries. + let shed = answered(Refused::Shed); + assert_eq!(shed.0, 429); + assert!(shed.1, "a shed says when to come back"); + assert!( + shed.2.contains(r#""code":"capacity_exhausted""#) && shed.2.contains(r#""type":"rate_limit_exceeded""#), + "{}", + shed.2 + ); + // An outage is not load, so it stays 503. + let outage = answered(Refused::NotReady); + assert_eq!(outage.0, 503); + assert!(outage.1); + assert!( + outage.2.contains(r#""code":"no_healthy_site""#) && outage.2.contains(r#""type":"server_error""#), + "{}", + outage.2 + ); + // An unknown model is the caller's error, with nothing to retry. + let unknown = answered(Refused::UnknownModel); + assert_eq!(unknown.0, 404); + assert!(!unknown.1, "a model that does not exist will not appear"); + assert!(unknown.2.contains(r#""code":"model_not_found""#), "{}", unknown.2); + } + + /// How `reason` is answered: status, whether it says when to retry, and its body. + fn answered(reason: Refused) -> (u16, bool, String) { + match refuse(reason, "llama", 0) { + FilterAction::TerminalResponse(response) => { + let body = String::from_utf8(response.body.clone().unwrap_or_default().to_vec()).unwrap(); + ( + response.status, + response.headers.contains_key(http::header::RETRY_AFTER), + body, + ) + }, + FilterAction::Continue + | FilterAction::Reject(_) + | FilterAction::StreamingTerminalResponse(_) + | FilterAction::Release + | FilterAction::BodyDone => panic!("a refusal is answered here"), + } + } + + #[test] + fn draws_are_uniform_and_seeds_differ() { + let mean = (0..10_000).map(|turn| unit(turn, SALTS.0)).sum::() / 10_000.0; + assert!((0.48..=0.52).contains(&mean), "{mean}"); + assert!((0..10_000).all(|turn| (0.0..1.0).contains(&unit(turn, SALTS.1)))); + assert_ne!(random_seed(), random_seed()); } } diff --git a/gateway/ai-grid-filters/src/serving.rs b/gateway/ai-grid-filters/src/serving.rs index e2aec6c3..bb531ca1 100644 --- a/gateway/ai-grid-filters/src/serving.rs +++ b/gateway/ai-grid-filters/src/serving.rs @@ -23,6 +23,7 @@ use serde::Deserialize; use crate::{ control::{Control, ReloadOutcome, Watcher, watch}, descriptor::CandidateConfig, + health::ClusterHealth, snapshot::RouteSnapshot, }; @@ -110,6 +111,10 @@ pub struct PeerServingConfig { /// Leaf SHA-256 digests the peer must also match, rendered under pin trust only. #[serde(default)] pub pins: Vec, + + /// `host:port` of the peer's gateway, routed to directly; set only while the operator verified it. + #[serde(default)] + pub gateway: Option, } /// The running control plane: the filter's snapshot, the pollers, and the config watch. @@ -122,6 +127,12 @@ pub struct GridRuntime { /// The config file watch, stopped on drop. watcher: Option, + + /// Backend cluster health the route filter publishes and the tick reads. + health: Arc, + + /// Stops the health tick when the runtime drops. + _health_tick: std::sync::mpsc::Sender<()>, } impl GridRuntime { @@ -131,6 +142,12 @@ impl GridRuntime { Arc::clone(&self.snapshot) } + /// The cluster health to register the filter over. + #[must_use] + pub fn health(&self) -> Arc { + Arc::clone(&self.health) + } + /// Apply a new serving config. `None` when it is already applied. /// /// # Errors @@ -180,6 +197,8 @@ pub fn load_serving_config(path: &str) -> Result /// peer's certificate material cannot be read or parsed, or a poller thread /// cannot be spawned. pub fn spawn_grid_routing(config: &GridServingConfig) -> Result { + // The peer scrapers load TLS here, before the server installs the provider. + praxis_tls::provider::install(); let start = Box::new(|peer: &PeerServingConfig, poller: &_, store, refresh| { let scraper = build_scraper(peer)?; spawn_on_thread_held(store, poller, scraper, refresh) @@ -195,15 +214,42 @@ pub(crate) fn start_runtime( ) -> Result { let mut control = Control::new(config, start)?; control.apply(config)?; + let health_tick = tick_health(control.health(), control.refresh(), control.store()) + .map_err(|error| -> FilterError { format!("grid: spawning the health tick: {error}").into() })?; Ok(GridRuntime { snapshot: control.snapshot(), + health: control.health(), control: Arc::new(Mutex::new(control)), watcher: None, + _health_tick: health_tick, }) } -/// Reject peer settings that would silently stop a poller. +/// How often the control step re-reads backend cluster health. +const HEALTH_TICK: Duration = Duration::from_millis(100); + +/// Re-order the snapshot whenever the set of clusters with no healthy endpoint changes. /// +/// Health moves faster than the poll cycle, so it gets its own tick; the returned +/// sender stops the thread when dropped. +fn tick_health( + health: Arc, + refresh: crate::control::Refresh, + store: Arc, +) -> std::io::Result> { + let (stop, stopped) = std::sync::mpsc::channel::<()>(); + std::thread::Builder::new() + .name("grid-health".to_owned()) + .spawn(move || { + while let Err(std::sync::mpsc::RecvTimeoutError::Timeout) = stopped.recv_timeout(HEALTH_TICK) { + if health.update() { + refresh(&store); + } + } + })?; + Ok(stop) +} + /// A zero interval hands `Duration::ZERO` to the interval timer, which panics the /// detached poller thread. A zero timeout fires immediately, so the peer never /// scrapes. Either way that site ages to `+inf` and sorts last, a silent stale @@ -234,6 +280,7 @@ pub(crate) fn validate_peer(peer: &PeerServingConfig) -> Result<(), FilterError> Ok(()) } +/// Grid mTLS to `peer`: this site's identity, verified against the grid CA and the peer's server name. /// Build a peer's mTLS scraper from its config, reading and parsing its /// certificate material. fn build_scraper(peer: &PeerServingConfig) -> Result { @@ -347,6 +394,7 @@ peers: client_cert_path: "/etc/grid/tls.crt".to_owned(), client_key_path: "/etc/grid/tls.key".to_owned(), pins: Vec::new(), + gateway: None, } } @@ -412,4 +460,14 @@ peers: validate_peer(&valid_peer()).expect("a valid peer is accepted"); } + + #[test] + fn a_peer_from_an_operator_without_gateway_support_parses_with_none() { + let peer: PeerServingConfig = serde_yaml::from_str( + "{site: east, addr: '10.0.0.1:9091', server_name: east.grid.internal, authority: east.grid.internal, \ + grid_ca_path: /ca, client_cert_path: /crt, client_key_path: /key}", + ) + .expect("an old operator's peer parses"); + assert!(peer.gateway.is_none(), "no gateway: route through the static cluster"); + } } diff --git a/gateway/ai-grid-filters/src/snapshot.rs b/gateway/ai-grid-filters/src/snapshot.rs index 553f1e98..53395b3c 100644 --- a/gateway/ai-grid-filters/src/snapshot.rs +++ b/gateway/ai-grid-filters/src/snapshot.rs @@ -6,18 +6,60 @@ //! ([`RouteSnapshot::from_store`]) run off the request path, so the hot path //! snapshots resolved order once rather than reading raw signals per request. -use std::sync::Arc; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; use grid_signals::LoadStore; -use crate::descriptor::RouteCandidate; +use crate::{ + decisions::SiteDecisions, + descriptor::{AdmissionState, CapabilityKind, RouteCandidate}, +}; -/// Load metric that orders candidates. Lower queue depth is a better target. -pub(crate) const LOAD_METRIC: &str = "inference_pool_average_queue_size"; +/// Load metrics that order candidates, current EPP name first. Lower queue depth is a better target. +/// +/// The wire carries each backend's raw metric name, so the consumer maps every name it knows. +pub(crate) const LOAD_METRICS: [&str; 2] = ["llm_d_epp_average_queue_size", "inference_pool_average_queue_size"]; + +/// The load metric current llm-d EPPs export. +#[cfg(test)] +pub(crate) const LOAD_METRIC: &str = LOAD_METRICS[0]; + +/// The requests a provider runs at once, as its operator publishes it. +const CAPACITY_METRIC: &str = "grid_provider_capacity_requests"; + +/// Requests a provider holds over its capacity, as its operator resolves it. +const SATURATION_METRIC: &str = "grid_provider_saturation_ratio"; + +/// Recent latency a provider's operator publishes: TTFT median and 90th percentile, time per +/// output token, and prefill seconds per uncached token. +const LATENCY_METRICS: [&str; 4] = [ + "grid_provider_ttft_p50_seconds", + "grid_provider_ttft_p90_seconds", + "grid_provider_tpot_seconds", + "grid_provider_prefill_seconds_per_token", +]; + +/// A model sheds once every healthy site serving it holds at least this rho. +const SHED_AT: f64 = 1.05; + +/// A shedding model routes again once any healthy site is at or below this rho. +const RESUME_AT: f64 = 0.95; + +/// The operator's per-provider verdict: 0 when the provider cannot serve. +/// +/// Absent reads as ready, so a site whose operator predates it is still routed. +const READY_METRIC: &str = "grid_provider_ready"; /// Lower load is a better routing target. const LOWER_IS_BETTER: bool = true; +/// Score of a candidate whose cluster has no healthy endpoint: after every healthy one, +/// unmeasured included, since `total_cmp` orders a positive NaN above infinity. +const UNHEALTHY: f64 = f64::NAN.abs(); + /// A resolved, pre-ordered candidate list read atomically per request. #[derive(Debug)] pub struct RouteSnapshot { @@ -26,6 +68,15 @@ pub struct RouteSnapshot { /// This gateway's own site identifier. pub local_site: Arc, + + /// Each candidate's score, parallel to `candidates`; equal scores share traffic. + pub scores: Vec, + + /// Each candidate's site decision counters, parallel to `candidates`. + pub decisions: Vec, + + /// Models answering 503 because every healthy site serving them is past full. + pub shedding: BTreeSet>, } impl RouteSnapshot { @@ -34,7 +85,14 @@ impl RouteSnapshot { /// The order is whatever the caller supplies (config order). Used before /// any signals exist and as the cold-start fallback. pub fn from_static(candidates: Vec, local_site: Arc) -> Self { - Self { candidates, local_site } + let scores = vec![f64::INFINITY; candidates.len()]; + Self { + decisions: decisions_for(&candidates), + candidates, + local_site, + scores, + shedding: BTreeSet::new(), + } } /// Order candidates least-loaded-first from the live store, then wrap them. @@ -54,15 +112,162 @@ impl RouteSnapshot { ) -> Self { // Score each candidate once, then sort the pairs: load_of allocates a // store key and scans a window, too costly to repeat inside sort_by. - let mut scored: Vec<(f64, RouteCandidate)> = candidates + let ranked = candidates .into_iter() - .map(|candidate| (Self::load_of(store, &candidate, now_ms, window_ms), candidate)) + .map(|mut candidate| { + if Self::reported_unready(store, &candidate) { + candidate.admission_state = AdmissionState::Excluded; + } + candidate.capacity = Self::capacity_of(store, &candidate); + candidate.rho = Self::fresh(store, &candidate, SATURATION_METRIC, now_ms, window_ms); + let [ttft_p50, ttft_p90, tpot, prefill] = + LATENCY_METRICS.map(|metric| Self::fresh(store, &candidate, metric, now_ms, window_ms)); + candidate.latency = crate::descriptor::Latency { + ttft_p50, + ttft_p90, + tpot, + prefill_per_token: prefill, + }; + let load = if candidate.admission_state == AdmissionState::NewAndExisting { + Self::load_of(store, &candidate, now_ms, window_ms) + } else { + UNHEALTHY + }; + (load, candidate) + }) .collect(); - scored.sort_by(|(left, _), (right, _)| left.total_cmp(right)); - let ordered = scored.into_iter().map(|(_, candidate)| candidate).collect(); + Self::ranked(ranked, local_site) + } + + /// The candidate's published capacity, `None` when none is published or it is not a positive count. + fn capacity_of(store: &LoadStore, candidate: &RouteCandidate) -> Option { + let key = LoadStore::key(&candidate.site, &candidate.cluster); + store + .latest(&key, CAPACITY_METRIC) + .map(|sample| sample.value) + .filter(|capacity| capacity.is_finite() && *capacity >= 1.0) + } + + /// The candidate's latest `metric` sample, `None` when absent or older than `window_ms`. + fn fresh(store: &LoadStore, candidate: &RouteCandidate, metric: &str, now_ms: i64, window_ms: i64) -> Option { + let key = LoadStore::key(&candidate.site, &candidate.cluster); + store + .latest(&key, metric) + .filter(|sample| now_ms.saturating_sub(sample.at_ms) <= window_ms && sample.value.is_finite()) + .map(|sample| sample.value.max(0.0)) + } + + /// This snapshot with the models to shed, given the set `previous` shed. + /// + /// A model sheds when every admitted, healthy site serving it has a fresh rho of at + /// least 1.05, and routes again once one reaches 0.95. Between the two it keeps its + /// previous state, so a value near the threshold does not flap. A model with any + /// healthy site of unknown rho never sheds: that site takes the overflow. + #[must_use] + pub(crate) fn shed(mut self, previous: &BTreeSet>) -> Self { + // Per model: whether every healthy site is measured, and their rho. + let mut models: BTreeMap<&Arc, (bool, Vec)> = BTreeMap::new(); + let healthy = self.candidates.iter().zip(&self.scores).filter(|(candidate, score)| { + candidate.kind == CapabilityKind::InferenceModel + && candidate.admission_state == AdmissionState::NewAndExisting + && !score.is_nan() + }); + for (candidate, _) in healthy { + let (measured, loads) = models.entry(&candidate.name).or_insert((true, Vec::new())); + match candidate.rho { + Some(rho) => loads.push(rho), + None => *measured = false, + } + } + self.shedding = models + .into_iter() + .filter(|(model, (measured, loads))| { + let full = loads.iter().all(|rho| *rho >= SHED_AT); + let room = loads.iter().any(|rho| *rho <= RESUME_AT); + *measured && !loads.is_empty() && (full || (previous.contains(*model) && !room)) + }) + .map(|(model, _)| Arc::clone(model)) + .collect(); + self + } + + /// Whether the candidate's latest readiness sample says it cannot serve. + /// + /// The latest at any age, not the worst in a window: a recovered provider rejoins + /// on its next poll, and a partitioned peer keeps its last 0 rather than reading as + /// ready once it ages out. A 0 stands until something newer arrives for the + /// provider: a later reading without the series means its peer stopped publishing + /// readiness, which reads as ready, like a peer that never published it. + fn reported_unready(store: &LoadStore, candidate: &RouteCandidate) -> bool { + let key = LoadStore::key(&candidate.site, &candidate.cluster); + store + .latest(&key, READY_METRIC) + .is_some_and(|ready| ready.value < 1.0 && store.newest_at(&key).is_none_or(|newest| newest <= ready.at_ms)) + } + + /// This snapshot with every candidate on a `down` cluster ordered after all the rest. + /// + /// Last rather than dropped: while any candidate is healthy it is never chosen, and + /// when none is, the request still has somewhere to go. + #[must_use] + pub(crate) fn demote(self, down: &BTreeSet>) -> Self { + self.demote_where(down, |candidate| Some(&candidate.cluster)) + } + + /// This snapshot with every candidate whose `key` is in `down` ordered last. + fn demote_where(self, down: &BTreeSet>, key: impl Fn(&RouteCandidate) -> Option<&Arc>) -> Self { + if down.is_empty() { + return self; + } + let ranked = self + .scores + .into_iter() + .zip(self.candidates) + .map(|(load, candidate)| { + let demoted = if key(&candidate).is_some_and(|key| down.contains(key)) { + UNHEALTHY + } else { + load + }; + (demoted, candidate) + }) + .collect(); + Self::ranked(ranked, self.local_site) + } + + /// This snapshot, after setting `grid_route_site_score` for each of its site/cluster pairs. + /// + /// The gauge is the score the order used, NaN when the candidate is excluded or + /// demoted. A pair in `published` but no longer in the topology is set to NaN, + /// since the exporter keeps a series until restart. `published` becomes this + /// snapshot's pairs. + #[must_use] + pub(crate) fn published(self, published: &mut Gauged) -> Self { + let mut current = BTreeSet::new(); + for (candidate, score) in self.candidates.iter().zip(&self.scores) { + let key = (Arc::clone(&candidate.site), Arc::clone(&candidate.cluster)); + // The front entry for a pair is its best; later ones are other models. + if current.insert(key.clone()) { + site_score(&key, *score); + } + } + for gone in published.difference(¤t) { + site_score(gone, f64::NAN); + } + *published = current; + self + } + + /// Sort `ranked` ascending by score, stable among equals, and wrap it. + fn ranked(mut ranked: Vec<(f64, RouteCandidate)>, local_site: Arc) -> Self { + ranked.sort_by(|(left, _), (right, _)| left.total_cmp(right)); + let (scores, ordered): (Vec, Vec) = ranked.into_iter().unzip(); Self { + decisions: decisions_for(&ordered), candidates: ordered, local_site, + scores, + shedding: BTreeSet::new(), } } @@ -71,12 +276,32 @@ impl RouteSnapshot { /// `+inf` makes an unmeasured candidate sort last under an ascending order. fn load_of(store: &LoadStore, candidate: &RouteCandidate, now_ms: i64, window_ms: i64) -> f64 { let key = LoadStore::key(&candidate.site, &candidate.cluster); - store - .window_worst(&key, LOAD_METRIC, now_ms, window_ms, LOWER_IS_BETTER) + LOAD_METRICS + .iter() + .find_map(|metric| store.window_worst(&key, metric, now_ms, window_ms, LOWER_IS_BETTER)) .unwrap_or(f64::INFINITY) } } +/// Decision counters for each of `candidates`, in order. +fn decisions_for(candidates: &[RouteCandidate]) -> Vec { + candidates + .iter() + .map(|candidate| SiteDecisions::new(&candidate.site)) + .collect() +} + +/// A candidate's site and cluster, the labels of its score gauge. +type SitePair = (Arc, Arc); + +/// The site/cluster pairs with a published score gauge. +pub(crate) type Gauged = BTreeSet; + +/// Set `grid_route_site_score` for one site/cluster pair. +fn site_score((site, cluster): &SitePair, score: f64) { + metrics::gauge!("grid_route_site_score", "site" => Arc::clone(site), "cluster" => Arc::clone(cluster)).set(score); +} + #[cfg(test)] #[expect(clippy::allow_attributes, reason = "blanket test suppressions")] #[allow( @@ -85,13 +310,14 @@ impl RouteSnapshot { clippy::indexing_slicing, clippy::panic, clippy::min_ident_chars, + clippy::too_many_lines, reason = "tests" )] mod tests { use std::time::Duration; use super::*; - use crate::descriptor::{CandidateConfig, CapabilityKind, validate_candidates}; + use crate::descriptor::{CandidateConfig, validate_candidates}; /// One QUEUE sample for `site`/`cluster` at `at_ms`. fn line(site: &str, cluster: &str, value: f64, at_ms: i64) -> String { @@ -101,6 +327,7 @@ mod tests { /// A validated candidate for model `name` served by `site`/`cluster`. fn cand(name: &str, site: &str, cluster: &str) -> CandidateConfig { CandidateConfig { + admission: AdmissionState::default(), cluster: cluster.to_owned(), credential: None, fresh: true, @@ -110,6 +337,188 @@ mod tests { } } + #[test] + fn a_cluster_with_no_healthy_endpoint_is_never_chosen_while_another_is_healthy() { + let store = LoadStore::new(Duration::from_secs(60)); + // site-a is the least loaded but down; site-d is unmeasured. + store.ingest_at(&line("local", "site-a", 0.0, 1_000), 1_000, 1_000, "local"); + store.ingest_at(&line("local", "site-b", 50.0, 1_000), 1_000, 1_000, "local"); + let candidates = validate_candidates(vec![ + cand("m", "local", "site-a"), + cand("m", "local", "site-b"), + cand("m", "site-d", "site-d"), + ]) + .unwrap(); + let down = BTreeSet::from([Arc::from("site-a")]); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("local"), &store, 1_000, 60_000).demote(&down); + let order: Vec<&str> = snapshot.candidates.iter().map(|c| &*c.cluster).collect(); + assert_eq!(order, ["site-b", "site-d", "site-a"], "down sorts after unmeasured"); + for turn in 0..10 { + let picked = crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + turn, + |_| true, + &crate::route::KeepAll, + ) + .unwrap() + .candidate; + assert_ne!(&*picked.cluster, "site-a", "turn {turn}"); + } + } + + /// One readiness sample for `site`/`cluster` at `at_ms`. + fn ready(site: &str, cluster: &str, value: f64, at_ms: i64) -> String { + format!(r#"{READY_METRIC}{{grid_site="{site}",grid_provider="{cluster}"}} {value} {at_ms}"#) + } + + #[test] + fn a_provider_reported_unready_is_excluded_until_a_newer_sample_says_ready() { + let store = LoadStore::new(Duration::from_secs(60)); + // site-b is the least loaded, but its operator says it cannot serve. + store.ingest_at(&line("site-a", "pool-a", 5.0, 1_000), 1_000, 1_000, "site-a"); + store.ingest_at(&line("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + store.ingest_at(&ready("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + let candidates = || { + validate_candidates(vec![ + cand("m", "site-a", "pool-a"), + cand("m", "site-b", "pool-b"), + cand("m", "site-d", "pool-d"), + ]) + .unwrap() + }; + let snapshot = RouteSnapshot::from_store(candidates(), Arc::from("hub"), &store, 1_000, 30_000); + let order: Vec<&str> = snapshot.candidates.iter().map(|c| &*c.cluster).collect(); + assert_eq!(order, ["pool-a", "pool-d", "pool-b"], "unready sorts last"); + for turn in 0..10 { + let picked = crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + turn, + |_| true, + &crate::route::KeepAll, + ) + .unwrap() + .candidate; + assert_ne!(&*picked.cluster, "pool-b", "turn {turn}"); + } + + store.ingest_at(&ready("site-b", "pool-b", 1.0, 2_000), 2_000, 2_000, "site-b"); + let recovered = RouteSnapshot::from_store(candidates(), Arc::from("hub"), &store, 2_000, 30_000); + assert_eq!( + &*recovered.candidates[0].cluster, "pool-b", + "back on the next sample, not after the window" + ); + } + + #[test] + fn a_not_ready_verdict_holds_through_silence_and_clears_when_the_peer_stops_publishing_it() { + let store = LoadStore::new(Duration::from_secs(60)); + store.ingest_at(&line("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + store.ingest_at(&ready("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + let order = |now| { + let candidates = + validate_candidates(vec![cand("m", "site-b", "pool-b"), cand("m", "site-a", "pool-a")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, now, 30_000); + snapshot.candidates.first().map(|c| c.cluster.to_string()) + }; + // Partitioned: nothing newer arrives, long after the load window. + assert_eq!( + order(120_000).as_deref(), + Some("pool-a"), + "the last 0 stands through silence" + ); + // Healed, but the peer no longer publishes readiness. + store.ingest_at(&line("site-b", "pool-b", 0.0, 121_000), 121_000, 121_000, "site-b"); + assert_eq!( + order(121_000).as_deref(), + Some("pool-b"), + "a newer reading without the series is ready" + ); + } + + #[test] + fn capacity_comes_from_the_published_series_and_is_unknown_otherwise() { + let store = LoadStore::new(Duration::from_secs(60)); + let capacity = |site: &str, cluster: &str, value: f64| { + format!(r#"grid_provider_capacity_requests{{grid_site="{site}",grid_provider="{cluster}"}} {value} 1000"#) + }; + store.ingest_at(&capacity("site-a", "pool-a", 64.0), 1_000, 1_000, "site-a"); + store.ingest_at(&capacity("site-b", "pool-b", 0.0), 1_000, 1_000, "site-b"); + let candidates = validate_candidates(vec![ + cand("m", "site-a", "pool-a"), + cand("m", "site-b", "pool-b"), + cand("m", "site-d", "pool-d"), + ]) + .unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000); + let by_site: Vec<(&str, Option)> = snapshot.candidates.iter().map(|c| (&*c.site, c.capacity)).collect(); + assert!(by_site.contains(&("site-a", Some(64.0)))); + assert!(by_site.contains(&("site-b", None)), "0 is not a capacity"); + assert!(by_site.contains(&("site-d", None)), "none published"); + } + + #[test] + fn an_excluded_candidate_from_the_serving_config_is_never_chosen() { + let store = LoadStore::new(Duration::from_secs(60)); + let mut excluded = cand("m", "site-b", "pool-b"); + excluded.admission = AdmissionState::Excluded; + let candidates = validate_candidates(vec![excluded, cand("m", "site-a", "pool-a")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000); + for turn in 0..4 { + let picked = crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + turn, + |_| true, + &crate::route::KeepAll, + ) + .unwrap() + .candidate; + assert_eq!(&*picked.cluster, "pool-a"); + } + } + + #[test] + fn when_every_cluster_is_down_one_is_still_chosen() { + let store = LoadStore::new(Duration::from_secs(60)); + let candidates = validate_candidates(vec![cand("m", "local", "site-a"), cand("m", "local", "site-b")]).unwrap(); + let down = BTreeSet::from([Arc::from("site-a"), Arc::from("site-b")]); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("local"), &store, 1_000, 60_000).demote(&down); + assert!( + crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + 0, + |_| true, + &crate::route::KeepAll + ) + .is_some() + ); + } + + #[test] + fn a_site_reporting_the_legacy_load_name_is_still_ordered() { + let store = LoadStore::new(Duration::from_secs(60)); + let legacy = |site: &str, cluster: &str, value: f64| { + format!(r#"inference_pool_average_queue_size{{grid_site="{site}",grid_provider="{cluster}"}} {value} 1000"#) + }; + store.ingest_at(&legacy("east", "pool-a", 90.0), 1_000, 1_000, "east"); + store.ingest_at(&line("west", "pool-b", 10.0, 1_000), 1_000, 1_000, "west"); + let candidates = validate_candidates(vec![cand("m", "east", "pool-a"), cand("m", "west", "pool-b")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("local"), &store, 1_000, 60_000); + let order: Vec<&str> = snapshot.candidates.iter().map(|c| &*c.site).collect(); + assert_eq!( + order, + ["west", "east"], + "both names measure load; neither sorts as unmeasured" + ); + } + #[test] fn the_least_loaded_site_sorts_first() { let store = LoadStore::new(Duration::from_secs(60)); @@ -159,4 +568,79 @@ mod tests { ); assert_eq!(&*snap.candidates[1].site, "west"); } + + /// One snapshot of sites `a` and `b` at the given rho, with capacity 100. + fn at_rho(rhos: [Option; 2]) -> RouteSnapshot { + let store = LoadStore::new(Duration::from_secs(60)); + for (site, rho) in ["a", "b"].iter().zip(rhos) { + let labels = format!(r#"grid_site="{site}",grid_provider="pool-{site}""#); + store.ingest_at( + &format!("grid_provider_capacity_requests{{{labels}}} 100 1000"), + 1_000, + 1_000, + site, + ); + if let Some(rho) = rho { + store.ingest_at( + &format!("grid_provider_saturation_ratio{{{labels}}} {rho} 1000"), + 1_000, + 1_000, + site, + ); + } + } + let candidates = validate_candidates(vec![cand("m", "a", "pool-a"), cand("m", "b", "pool-b")]).unwrap(); + RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000) + } + + #[test] + fn a_model_sheds_past_full_and_routes_again_once_a_site_has_room() { + let none = BTreeSet::new(); + let shed = BTreeSet::from([Arc::from("m")]); + assert!( + at_rho([Some(1.10), Some(1.06)]).shed(&none).shedding.contains("m"), + "both past 1.05" + ); + assert!( + at_rho([Some(1.10), Some(1.0)]).shed(&none).shedding.is_empty(), + "one below 1.05 does not start a shed" + ); + assert!( + at_rho([Some(1.10), Some(1.0)]).shed(&shed).shedding.contains("m"), + "nor does it end one" + ); + assert!( + at_rho([Some(1.10), Some(0.95)]).shed(&shed).shedding.is_empty(), + "0.95 ends it" + ); + assert!( + at_rho([Some(1.10), None]).shed(&shed).shedding.is_empty(), + "an unmeasured site never sheds" + ); + } + + #[test] + fn published_latency_reaches_the_candidate_and_absent_stays_unknown() { + let store = LoadStore::new(Duration::from_secs(60)); + let labels = r#"grid_site="a",grid_provider="pool-a""#; + store.ingest_at( + &format!("grid_provider_ttft_p90_seconds{{{labels}}} 0.8 1000"), + 1_000, + 1_000, + "a", + ); + store.ingest_at( + &format!("grid_provider_tpot_seconds{{{labels}}} 0.02 1000"), + 1_000, + 1_000, + "a", + ); + let candidates = validate_candidates(vec![cand("m", "a", "pool-a")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000); + let latency = snapshot.candidates[0].latency; + assert_eq!(latency.ttft_p90, Some(0.8)); + assert_eq!(latency.tpot, Some(0.02)); + assert_eq!(latency.ttft_p50, None, "unpublished"); + assert_eq!(latency.prefill_per_token, None, "unpublished"); + } } diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 629a2da2..8f90299e 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -17,6 +17,8 @@ use std::process::ExitCode; use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; use tracing::info; +mod metrics_listener; + /// Log line emitted once tracing is up; the startup test waits for it. const STARTUP_MESSAGE: &str = "starting grid-gateway"; @@ -42,6 +44,11 @@ fn main() -> ExitCode { let log_output = config.runtime.logging.output; info!(version = env!("CARGO_PKG_VERSION"), "{STARTUP_MESSAGE}"); + // Before grid routing starts, so its metrics record into the installed recorder. + if let Err(err) = start_metrics_listener(&config) { + return praxis::report_fatal(&err, log_output); + } + let mut registry = praxis_filter::FilterRegistry::with_builtins(); praxis_ai_filters::register_ai_filters(&mut registry, None); @@ -65,6 +72,27 @@ fn main() -> ExitCode { result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS) } +/// Start the opt-in metrics listener when its env vars are set. +/// +/// # Errors +/// +/// Returns the settings, port, cert, or bind error. +fn start_metrics_listener(config: &Config) -> Result<(), String> { + let listener = metrics_listener::MetricsListener::from_env(|name| std::env::var(name).ok())?; + // Praxis installs the recorder only when the admin server starts, after grid routing + // publishes its first snapshot, so install it now when anything will serve metrics. + if listener.is_some() || config.admin.address.is_some() { + praxis_protocol::http::pingora::metrics::install_prometheus_recorder(); + } + let Some(listener) = listener else { + return Ok(()); + }; + listener.check_ports(config)?; + // The thread serves for the life of the process. + drop(listener.spawn()?); + Ok(()) +} + /// Start the cross-site pollers and register `grid_site_route` over their snapshot. /// /// # Errors @@ -77,7 +105,7 @@ fn start_grid_routing( ) -> Result { let config = ai_grid_filters::load_serving_config(path)?; let mut runtime = ai_grid_filters::spawn_grid_routing(&config)?; - ai_grid_filters::register_grid_filters(registry, runtime.snapshot())?; + ai_grid_filters::register_grid_filters(registry, runtime.snapshot(), runtime.health())?; // The operator rewrites the file on membership and topology changes. runtime .watch(path, SERVING_RELOAD_INTERVAL) diff --git a/gateway/src/metrics_listener.rs b/gateway/src/metrics_listener.rs new file mode 100644 index 00000000..49a1c030 --- /dev/null +++ b/gateway/src/metrics_listener.rs @@ -0,0 +1,471 @@ +//! Opt-in TLS listener that serves only `GET /metrics`. +//! +//! The Praxis admin listener also serves kv, pipelines, and log-level with no +//! auth, and refuses a non-loopback `Host`, so an in-cluster scraper cannot use +//! it. This listener renders the same Prometheus registry on its own port. + +use std::{ + borrow::Cow, + io, + net::{SocketAddr, TcpListener as StdTcpListener}, + sync::Arc, + thread::JoinHandle, + time::Duration, +}; + +use praxis_core::config::Config; +use praxis_tls::{CertKeyPair, reload::ReloadableCertResolver, watcher::CertWatcher}; +use tokio::{ + io::{AsyncRead, AsyncReadExt as _, AsyncWriteExt as _}, + net::{TcpListener, TcpStream}, + sync::Semaphore, + time::timeout, +}; +use tokio_rustls::{ + TlsAcceptor, + rustls::{ServerConfig, crypto::CryptoProvider}, +}; +use tracing::{debug, info, warn}; + +/// Env var naming the listener's bind address. Unset leaves the listener off. +const ADDR_ENV: &str = "GRID_METRICS_ADDR"; + +/// Env var naming the serving certificate (PEM chain). +const CERT_ENV: &str = "GRID_METRICS_TLS_CERT"; + +/// Env var naming the serving key (PEM). +const KEY_ENV: &str = "GRID_METRICS_TLS_KEY"; + +/// Concurrent connections. A scraper needs one, and extras are closed on accept. +const MAX_CONNECTIONS: usize = 4; + +/// Deadline for the TLS handshake. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); + +/// Deadline for a whole connection, handshake to close, so no client holds a slot longer. +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); + +/// Largest request head read. A longer one gets 431. +const MAX_HEAD: usize = 8 * 1024; + +/// Most request headers parsed. +const MAX_HEADERS: usize = 32; + +/// Pause after a failed accept, so a persistent error such as EMFILE does not spin. +const ACCEPT_BACKOFF: Duration = Duration::from_millis(100); + +/// The listener's settings, read from the environment. +#[derive(Debug)] +pub(crate) struct MetricsListener { + /// Bind address. + addr: SocketAddr, + + /// Serving certificate and key, reloaded on change. + pair: CertKeyPair, +} + +impl MetricsListener { + /// The listener settings from `lookup`, or `None` when the address is unset. + /// + /// # Errors + /// + /// Returns a message for an unparsable address or a missing cert or key. + pub(crate) fn from_env Option>(lookup: F) -> Result, String> { + let Some(addr) = lookup(ADDR_ENV).filter(|addr| !addr.is_empty()) else { + return Ok(None); + }; + let addr = addr + .parse() + .map_err(|err| format!("{ADDR_ENV} {addr:?} is not an ip:port: {err}"))?; + let path = |name: &str| { + lookup(name) + .filter(|path| !path.is_empty()) + .ok_or_else(|| format!("{ADDR_ENV} needs {name}: the metrics listener serves TLS only")) + }; + Ok(Some(Self { + addr, + pair: CertKeyPair { + cert_path: path(CERT_ENV)?, + default: false, + key_path: path(KEY_ENV)?, + server_names: Vec::new(), + }, + })) + } + + /// Refuse a port that a Praxis listener or the admin listener already binds. + /// + /// # Errors + /// + /// Returns a message naming the colliding address, or one with no readable port. + pub(crate) fn check_ports(&self, config: &Config) -> Result<(), String> { + let taken = config + .listeners + .iter() + .map(|listener| listener.address.as_str()) + .chain(config.admin.address.as_deref()); + for address in taken { + let port = address + .rsplit_once(':') + .and_then(|(_, port)| port.parse::().ok()) + .ok_or_else(|| format!("Praxis address {address:?} has no port to check {ADDR_ENV} against"))?; + if port == self.addr.port() { + return Err(format!( + "{ADDR_ENV} {} collides with Praxis address {address}", + self.addr + )); + } + } + Ok(()) + } + + /// Bind the listener and serve it on its own thread. + /// + /// # Errors + /// + /// Returns a message when the cert does not load, no crypto provider is + /// installed, or the address does not bind. + pub(crate) fn spawn(self) -> Result, String> { + let resolver = ReloadableCertResolver::new(&self.pair) + .map_err(|err| format!("metrics listener certificate {}: {err}", self.pair.cert_path))?; + let current = resolver.arc(); + let acceptor = TlsAcceptor::from(Arc::new(server_config(resolver)?)); + let listener = StdTcpListener::bind(self.addr) + .and_then(|listener| listener.set_nonblocking(true).map(|()| listener)) + .map_err(|err| format!("metrics listener bind {}: {err}", self.addr))?; + // A dropped sender leaves the watcher running for the life of the process. + let (_, shutdown) = tokio::sync::watch::channel(false); + drop(CertWatcher::spawn(current, self.pair, None, shutdown)); + let addr = self.addr; + std::thread::Builder::new() + .name("grid-metrics".to_owned()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime, + Err(err) => { + warn!(error = %err, "metrics listener runtime failed to start"); + return; + }, + }; + info!(%addr, "metrics listener serving"); + runtime.block_on(serve(listener, acceptor)); + }) + .map_err(|err| format!("metrics listener thread: {err}")) + } +} + +/// The listener's TLS config, over the process crypto provider so FIPS mode carries over. +/// +/// # Errors +/// +/// Returns a message when no provider is installed, it rejects the protocol versions, +/// or FIPS is required and the config is not FIPS. +fn server_config(resolver: ReloadableCertResolver) -> Result { + let provider = CryptoProvider::get_default() + .ok_or_else(|| "metrics listener: no rustls crypto provider installed".to_owned())?; + let mut config = ServerConfig::builder_with_provider(Arc::clone(provider)) + .with_safe_default_protocol_versions() + .map_err(|err| format!("metrics listener TLS: {err}"))? + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver)); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + // As Praxis sets on its own listeners: SP 800-52r2 needs EMS for TLS 1.2. + config.require_ems = true; + if praxis_tls::provider::required() && !config.fips() { + return Err("metrics listener: FIPS is required but its TLS config is not FIPS".to_owned()); + } + Ok(config) +} + +/// Register the bound socket with the runtime and serve it. +async fn serve(listener: StdTcpListener, acceptor: TlsAcceptor) { + match TcpListener::from_std(listener) { + Ok(listener) => accept_loop(&listener, &acceptor).await, + Err(err) => warn!(error = %err, "metrics listener failed to register its socket"), + } +} + +/// Accept connections forever, closing any beyond [`MAX_CONNECTIONS`]. +#[expect(clippy::infinite_loop, reason = "serves for the life of the process")] +async fn accept_loop(listener: &TcpListener, acceptor: &TlsAcceptor) { + let slots = Arc::new(Semaphore::new(MAX_CONNECTIONS)); + loop { + let (stream, peer) = match listener.accept().await { + Ok(accepted) => accepted, + Err(err) => { + warn!(error = %err, "metrics listener accept failed"); + tokio::time::sleep(ACCEPT_BACKOFF).await; + continue; + }, + }; + let Ok(permit) = Arc::clone(&slots).try_acquire_owned() else { + debug!(%peer, "metrics listener at capacity, closing connection"); + drop(stream); + continue; + }; + let acceptor = acceptor.clone(); + drop(tokio::spawn(async move { + if let Err(err) = Box::pin(handle(stream, &acceptor)).await { + debug!(%peer, error = %err, "metrics connection failed"); + } + drop(permit); + })); + } +} + +/// Serve one request on `stream` and close it. +/// +/// # Errors +/// +/// Returns the I/O error or timeout that ended the connection. +async fn handle(stream: TcpStream, acceptor: &TlsAcceptor) -> io::Result<()> { + // Boxed: the TLS stream makes this future several KiB. + within( + CONNECTION_TIMEOUT, + Box::pin(async { + let mut tls = within(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await?; + let (head, body) = read_request(&mut tls).await?.into_parts(); + tls.write_all(head.as_bytes()).await?; + tls.write_all(body.as_bytes()).await?; + tls.shutdown().await + }), + ) + .await +} + +/// `future`'s result, or a `TimedOut` error after `limit`. +/// +/// # Errors +/// +/// Returns `future`'s error, or `TimedOut`. +async fn within>>(limit: Duration, future: F) -> io::Result { + timeout(limit, future) + .await + .unwrap_or_else(|_elapsed| Err(io::ErrorKind::TimedOut.into())) +} + +/// Read a request head of at most [`MAX_HEAD`] bytes and decide the reply. +/// +/// # Errors +/// +/// Returns the read error, or `UnexpectedEof` when the peer closes mid-head. +async fn read_request(stream: &mut S) -> io::Result { + let mut head = vec![0_u8; MAX_HEAD]; + let mut filled = 0_usize; + loop { + let Some(spare) = head.get_mut(filled..).filter(|spare| !spare.is_empty()) else { + return Ok(Reply::HeadTooLarge); + }; + let read = stream.read(spare).await?; + if read == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + filled = filled.checked_add(read).unwrap_or(MAX_HEAD); + let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut request = httparse::Request::new(&mut headers); + match request.parse(head.get(..filled).unwrap_or_default()) { + Ok(httparse::Status::Complete(_)) => return Ok(Reply::route(request.method, request.path)), + Ok(httparse::Status::Partial) => {}, + Err(httparse::Error::TooManyHeaders) => return Ok(Reply::HeadTooLarge), + Err(_) => return Ok(Reply::BadRequest), + } + } +} + +/// The listener's possible answers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Reply { + /// 200 with the Prometheus exposition, or 503 before the recorder exists. + Metrics, + /// 404 for any path but `/metrics`. + NotFound, + /// 405 for `/metrics` with a method other than GET. + MethodNotAllowed, + /// 400 for an unparsable head. + BadRequest, + /// 431 for a head over [`MAX_HEAD`] or [`MAX_HEADERS`]. + HeadTooLarge, +} + +impl Reply { + /// The reply for `method` and `path`, ignoring a query string on `/metrics`. + fn route(method: Option<&str>, path: Option<&str>) -> Self { + let path = path.map(|path| path.split_once('?').map_or(path, |(path, _)| path)); + match (method, path) { + (Some("GET"), Some("/metrics")) => Self::Metrics, + (_, Some("/metrics")) => Self::MethodNotAllowed, + _ => Self::NotFound, + } + } + + /// The HTTP/1.1 response head and body, closing the connection. + fn into_parts(self) -> (String, Cow<'static, str>) { + let (status, body) = match self { + Self::Metrics => { + return match praxis_protocol::http::pingora::metrics::render_prometheus() { + Some(body) => response("200 OK", "", "text/plain; version=0.0.4; charset=utf-8", body.into()), + None => response( + "503 Service Unavailable", + "", + "text/plain", + "metrics recorder not installed\n".into(), + ), + }; + }, + Self::NotFound => ("404 Not Found", "not found\n"), + Self::MethodNotAllowed => ("405 Method Not Allowed", "method not allowed\n"), + Self::BadRequest => ("400 Bad Request", "bad request\n"), + Self::HeadTooLarge => ("431 Request Header Fields Too Large", "request head too large\n"), + }; + let allow = if self == Self::MethodNotAllowed { + "Allow: GET\r\n" + } else { + "" + }; + response(status, allow, "text/plain", body.into()) + } +} + +/// An HTTP/1.1 response head with `extra` header lines, closing the connection, and its body. +fn response(status: &str, extra: &str, content_type: &str, body: Cow<'static, str>) -> (String, Cow<'static, str>) { + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n{extra}Connection: close\r\n\r\n", + body.len() + ); + (head, body) +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, reason = "tests")] +mod tests { + use std::path::PathBuf; + + use super::*; + + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let pairs: Vec<(String, String)> = pairs + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + move |name| { + pairs + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + } + } + + #[test] + fn an_unset_address_leaves_the_listener_off() { + assert!(MetricsListener::from_env(env(&[])).unwrap().is_none(), "unset"); + assert!( + MetricsListener::from_env(env(&[(ADDR_ENV, "")])).unwrap().is_none(), + "empty" + ); + } + + #[test] + fn an_address_without_a_cert_or_key_is_refused() { + let no_key = MetricsListener::from_env(env(&[(ADDR_ENV, "0.0.0.0:9443"), (CERT_ENV, "/c")])).unwrap_err(); + assert!(no_key.contains(KEY_ENV), "{no_key}"); + let no_cert = MetricsListener::from_env(env(&[(ADDR_ENV, "0.0.0.0:9443"), (KEY_ENV, "/k")])).unwrap_err(); + assert!(no_cert.contains(CERT_ENV), "{no_cert}"); + let hostname = MetricsListener::from_env(env(&[(ADDR_ENV, "metrics:9443")])).unwrap_err(); + assert!(hostname.contains("ip:port"), "{hostname}"); + } + + #[test] + fn a_port_praxis_binds_is_refused() { + let listener = MetricsListener::from_env(env(&[(ADDR_ENV, "0.0.0.0:9443"), (CERT_ENV, "/c"), (KEY_ENV, "/k")])) + .unwrap() + .unwrap(); + let config = |yaml: &str| Config::from_yaml(yaml).expect("config"); + let base = "listeners:\n - name: web\n address: \"0.0.0.0:8080\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters: []\n"; + listener.check_ports(&config(base)).unwrap(); + let clash = base.replace("8080", "9443"); + assert!( + listener.check_ports(&config(&clash)).unwrap_err().contains("collides"), + "listener" + ); + let admin = format!("{base}admin:\n address: \"127.0.0.1:9443\"\n"); + assert!( + listener.check_ports(&config(&admin)).unwrap_err().contains("collides"), + "admin" + ); + } + + async fn reply_to(request: &[u8]) -> Reply { + let mut reader = request; + read_request(&mut reader).await.unwrap() + } + + #[tokio::test] + async fn only_get_metrics_is_served() { + assert_eq!( + reply_to(b"GET /metrics HTTP/1.1\r\nHost: x\r\n\r\n").await, + Reply::Metrics + ); + assert_eq!(reply_to(b"GET /metrics?x=1 HTTP/1.1\r\n\r\n").await, Reply::Metrics); + assert_eq!( + reply_to(b"POST /metrics HTTP/1.1\r\n\r\n").await, + Reply::MethodNotAllowed + ); + assert_eq!( + reply_to(b"HEAD /metrics HTTP/1.1\r\n\r\n").await, + Reply::MethodNotAllowed + ); + for path in ["/", "/api/kv", "/ready", "/metrics/", "/metricsx"] { + let request = format!("GET {path} HTTP/1.1\r\n\r\n"); + assert_eq!(reply_to(request.as_bytes()).await, Reply::NotFound, "{path}"); + } + assert_eq!(reply_to(b"\x00garbage\r\n\r\n").await, Reply::BadRequest); + } + + #[tokio::test] + async fn an_oversized_head_gets_431_without_reading_past_the_cap() { + let mut long = b"GET /metrics HTTP/1.1\r\nX: ".to_vec(); + long.resize(MAX_HEAD.saturating_mul(2), b'a'); + assert_eq!(reply_to(&long).await, Reply::HeadTooLarge); + let many: String = std::iter::once("GET /metrics HTTP/1.1\r\n".to_owned()) + .chain((0..=MAX_HEADERS).map(|index| format!("H{index}: v\r\n"))) + .chain(std::iter::once("\r\n".to_owned())) + .collect(); + assert_eq!(reply_to(many.as_bytes()).await, Reply::HeadTooLarge); + } + + #[tokio::test] + async fn a_peer_closing_mid_head_is_an_error() { + let mut reader: &[u8] = b"GET /metrics HTTP/1.1\r\n"; + let err = read_request(&mut reader).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof, "{err}"); + } + + #[test] + fn the_tls_config_requires_extended_master_secret() { + praxis_tls::provider::install(); + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/tmp/metrics-listener-ems"); + std::fs::create_dir_all(&dir).unwrap(); + let ca = certs::generate_ca("ems test CA").unwrap(); + let leaf = certs::generate_site_cert(&ca, "metrics").unwrap(); + let pair = CertKeyPair { + cert_path: dir.join("tls.crt").display().to_string(), + default: false, + key_path: dir.join("tls.key").display().to_string(), + server_names: Vec::new(), + }; + std::fs::write(&pair.cert_path, &leaf.cert_pem).unwrap(); + std::fs::write(&pair.key_path, &leaf.key_pem).unwrap(); + let config = server_config(ReloadableCertResolver::new(&pair).unwrap()).unwrap(); + assert!(config.require_ems, "TLS 1.2 without EMS is outside SP 800-52r2"); + assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()], "HTTP/1.1 only"); + } + + #[test] + fn a_405_names_get_and_every_reply_closes() { + let (response, _) = Reply::MethodNotAllowed.into_parts(); + assert!(response.starts_with("HTTP/1.1 405 "), "{response}"); + assert!(response.contains("\r\nAllow: GET\r\n"), "{response}"); + assert!(response.contains("\r\nConnection: close\r\n"), "{response}"); + } +} diff --git a/gateway/tests/chart_renders.rs b/gateway/tests/chart_renders.rs new file mode 100644 index 00000000..4d00a1b3 --- /dev/null +++ b/gateway/tests/chart_renders.rs @@ -0,0 +1,116 @@ +//! The chart's default renders load in praxis, so a bad default fails here, not at pod start. +//! +//! Needs `helm` on PATH. Set `GRID_SKIP_CHART_RENDER=1` to skip where helm is absent; +//! otherwise a missing helm fails rather than passing vacuously. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::expect_used, clippy::panic, clippy::indexing_slicing, reason = "tests")] +mod tests { + use std::{ + path::PathBuf, + process::Command, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; + + /// Values every case starts from: a rendered gateway config with no auth. + const BASE: &[&str] = &[ + "--set=gatewayConfig.render=true", + "--set=gatewayConfig.localSite=hub", + "--set=gatewayConfig.model=qwen3", + "--set=gatewayConfig.auth.mode=none", + ]; + + /// The default local backend, a local tls workload with its CA, and a remote peer. + const CASES: &[(&str, &[&str])] = &[ + ( + "local plaintext", + &[ + "--set=gatewayConfig.backends[0].cluster=site-a", + "--set=gatewayConfig.backends[0].transport.mode=plaintext", + "--set=gatewayConfig.backends[0].endpoints[0]=10.0.0.5:8000", + ], + ), + ( + "local tls", + &[ + "--set=gatewayConfig.backends[0].cluster=kserve", + "--set=gatewayConfig.backends[0].transport.mode=tls", + "--set=gatewayConfig.backends[0].transport.sni=qwen3.ns.svc", + "--set=gatewayConfig.backends[0].transport.ca.configMap=service-ca", + "--set=gatewayConfig.backends[0].endpoints[0]=10.0.0.6:8443", + ], + ), + ( + "remote peer", + &[ + "--set=gatewayConfig.backends[0].cluster=pool-b", + "--set=gatewayConfig.backends[0].site=site-b", + "--set=gatewayConfig.backends[0].transport.mode=mutual_tls", + "--set=gatewayConfig.backends[0].transport.sni=site-b.grid.internal", + "--set=gatewayConfig.backends[0].endpoints[0]=203.0.113.7:8443", + "--set=tls.enabled=true", + "--set=tls.existingSecret=grid-site-identity", + "--set=tls.caSecret=grid-ca", + ], + ), + ]; + + /// The rendered praxis.yaml for `values`, or `None` when helm is absent and skipping is asked for. + fn render(values: &[&str]) -> Option { + let chart = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../charts/praxis-gateway"); + let output = match Command::new("helm") + .args(["template", "render"]) + .arg(&chart) + .args(BASE) + .args(values) + .args(["--show-only", "templates/gateway-config.yaml"]) + .output() + { + Ok(output) => output, + Err(_) if std::env::var("GRID_SKIP_CHART_RENDER").is_ok_and(|skip| skip == "1") => return None, + Err(error) => panic!("helm is needed to render the chart ({error}); set GRID_SKIP_CHART_RENDER=1 to skip"), + }; + assert!( + output.status.success(), + "helm template: {}", + String::from_utf8_lossy(&output.stderr) + ); + let config_map: serde_yaml::Value = serde_yaml::from_slice(&output.stdout).expect("a ConfigMap"); + Some( + config_map["data"]["praxis.yaml"] + .as_str() + .expect("praxis.yaml") + .to_owned(), + ) + } + + /// Load `yaml` the way the gateway does at startup. + fn load(yaml: &str) -> Result<(), String> { + // Tests run as threads of one process, so each load needs its own file. + static NEXT: AtomicUsize = AtomicUsize::new(0); + let name = format!( + "chart-render-{}-{}.yaml", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(name); + std::fs::write(&path, yaml).expect("write"); + let file = ConfigFile::read(&path).map_err(|error| error.to_string())?; + let loaded = Config::from_config_file_or(Some(&file), DEFAULT_CONFIG) + .map(drop) + .map_err(|error| error.to_string()); + let _removed = std::fs::remove_file(&path); + loaded + } + + #[test] + fn every_default_render_loads() { + for (name, values) in CASES { + let Some(yaml) = render(values) else { return }; + load(&yaml).unwrap_or_else(|error| panic!("{name}: {error}")); + } + } +} diff --git a/gateway/tests/metrics_listener.rs b/gateway/tests/metrics_listener.rs new file mode 100644 index 00000000..aa8184a9 --- /dev/null +++ b/gateway/tests/metrics_listener.rs @@ -0,0 +1,198 @@ +//! The opt-in metrics listener serves only `GET /metrics` over TLS, to any Host. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::disallowed_methods, + reason = "tests; waits poll a deadline with thread::sleep" +)] +mod tests { + use std::{ + io::{Read as _, Write as _}, + net::{TcpListener, TcpStream}, + path::PathBuf, + process::{Child, Command, Stdio}, + sync::Arc, + thread, + time::{Duration, Instant}, + }; + + use certs::{generate_ca, generate_site_cert}; + use rustls::{ + ClientConfig, ClientConnection, RootCertStore, StreamOwned, + pki_types::{CertificateDer, ServerName, pem::PemObject as _}, + }; + + /// Bound on how long a test waits for the child. + const TIMEOUT: Duration = Duration::from_secs(30); + + /// The SAN `generate_site_cert` gives site `metrics`. + const SERVER_NAME: &str = "metrics.grid.internal"; + + fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() + } + + /// A running gateway, killed on drop. + struct Gateway(Child); + + impl Drop for Gateway { + fn drop(&mut self) { + drop(self.0.kill()); + drop(self.0.wait()); + } + } + + /// Paths and client trust for one test's cert, key, and config. + struct Setup { + dir: PathBuf, + client: Arc, + listener_port: u16, + } + + fn setup(name: &str) -> Setup { + let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(name); + std::fs::create_dir_all(&dir).unwrap(); + let ca = generate_ca("metrics test CA").unwrap(); + let leaf = generate_site_cert(&ca, "metrics").unwrap(); + std::fs::write(dir.join("tls.crt"), &leaf.cert_pem).unwrap(); + std::fs::write(dir.join("tls.key"), &leaf.key_pem).unwrap(); + let listener_port = free_port(); + std::fs::write( + dir.join("praxis.yaml"), + format!( + "listeners:\n - name: default\n address: \"127.0.0.1:{listener_port}\"\n filter_chains: [main]\n\ + filter_chains:\n - name: main\n filters:\n - filter: static_response\n status: 200\n" + ), + ) + .unwrap(); + let mut roots = RootCertStore::empty(); + roots + .add(CertificateDer::from_pem_slice(ca.cert_pem.as_bytes()).unwrap()) + .unwrap(); + let client = ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + Setup { + dir, + client: Arc::new(client), + listener_port, + } + } + + fn start(setup: &Setup, metrics_port: u16) -> Gateway { + Gateway( + Command::new(env!("CARGO_BIN_EXE_grid-gateway")) + .arg("--config") + .arg(setup.dir.join("praxis.yaml")) + .env_remove("GRID_SERVING_CONFIG") + .env("GRID_METRICS_ADDR", format!("127.0.0.1:{metrics_port}")) + .env("GRID_METRICS_TLS_CERT", setup.dir.join("tls.crt")) + .env("GRID_METRICS_TLS_KEY", setup.dir.join("tls.key")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(), + ) + } + + /// Send `head` over TLS and return the full response, or `None` when the connection fails. + fn exchange(client: &Arc, port: u16, head: &str) -> Option { + let stream = TcpStream::connect(("127.0.0.1", port)).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(10))).ok()?; + let name = ServerName::try_from(SERVER_NAME).ok()?; + let conn = ClientConnection::new(Arc::clone(client), name).ok()?; + let mut tls = StreamOwned::new(conn, stream); + tls.write_all(head.as_bytes()).ok()?; + let mut response = String::new(); + tls.read_to_string(&mut response).ok()?; + Some(response) + } + + /// The response to `head`, retrying until the listener is up. + fn eventually(client: &Arc, port: u16, head: &str) -> String { + let deadline = Instant::now().checked_add(TIMEOUT).unwrap(); + loop { + if let Some(response) = exchange(client, port, head) { + return response; + } + assert!(Instant::now() < deadline, "metrics listener never answered on {port}"); + thread::sleep(Duration::from_millis(100)); + } + } + + #[test] + fn only_get_metrics_is_served_and_any_host_is_accepted() { + let setup = setup("metrics-listener-routes"); + let port = free_port(); + let _gateway = start(&setup, port); + + let metrics = eventually( + &setup.client, + port, + "GET /metrics HTTP/1.1\r\nHost: grid-gateway.grid-system.svc\r\n\r\n", + ); + assert!(metrics.starts_with("HTTP/1.1 200 "), "{metrics}"); + assert!(metrics.contains("text/plain; version=0.0.4"), "{metrics}"); + + let kv = eventually(&setup.client, port, "GET /api/kv HTTP/1.1\r\nHost: x\r\n\r\n"); + assert!(kv.starts_with("HTTP/1.1 404 "), "{kv}"); + let wrong_method = eventually( + &setup.client, + port, + "POST /metrics HTTP/1.1\r\nHost: x\r\nContent-Length: 0\r\n\r\n", + ); + assert!(wrong_method.starts_with("HTTP/1.1 405 "), "{wrong_method}"); + } + + #[test] + fn connections_past_the_cap_are_closed_and_slots_come_back() { + let setup = setup("metrics-listener-cap"); + let port = free_port(); + let _gateway = start(&setup, port); + eventually(&setup.client, port, "GET /metrics HTTP/1.1\r\n\r\n"); + + // Four idle connections hold every slot until the handshake deadline. + let held: Vec = std::iter::repeat_with(|| TcpStream::connect(("127.0.0.1", port)).unwrap()) + .take(4) + .collect(); + // Poll inside the 2s handshake deadline, which would free the held slots. + let deadline = Instant::now().checked_add(Duration::from_millis(1500)).unwrap(); + let refused = loop { + if exchange(&setup.client, port, "GET /metrics HTTP/1.1\r\n\r\n").is_none() { + break true; + } + if Instant::now() >= deadline { + break false; + } + thread::sleep(Duration::from_millis(50)); + }; + assert!(refused, "a fifth connection is closed"); + drop(held); + let after = eventually(&setup.client, port, "GET /metrics HTTP/1.1\r\n\r\n"); + assert!(after.starts_with("HTTP/1.1 200 "), "{after}"); + } + + #[test] + fn a_port_praxis_binds_stops_startup() { + let setup = setup("metrics-listener-collision"); + let mut gateway = start(&setup, setup.listener_port); + let deadline = Instant::now().checked_add(TIMEOUT).unwrap(); + let status = loop { + if let Some(status) = gateway.0.try_wait().unwrap() { + break status; + } + assert!( + Instant::now() < deadline, + "the gateway kept running on a colliding port" + ); + thread::sleep(Duration::from_millis(100)); + }; + assert!(!status.success(), "{status}"); + } +} diff --git a/gateway/tests/provider_readiness.rs b/gateway/tests/provider_readiness.rs new file mode 100644 index 00000000..bb684579 --- /dev/null +++ b/gateway/tests/provider_readiness.rs @@ -0,0 +1,512 @@ +//! A site whose operator reports its provider not ready gets no traffic, and rejoins when it recovers. +//! +//! Time to exclusion is one peer poll after the peer publishes `grid_provider_ready 0`, +//! and time to rejoin one poll after it publishes 1. With the operator's defaults (5s +//! scrape, 2 zero scrapes, 5s poll) that is at most about 15s to exclude and 10s to rejoin. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing, + clippy::min_ident_chars, + clippy::arithmetic_side_effects, + clippy::disallowed_methods, + clippy::too_many_lines, + reason = "tests; waits poll a deadline with thread::sleep" +)] +mod tests { + use std::{ + collections::BTreeMap, + io::{BufRead as _, BufReader, ErrorKind, Read as _, Write as _}, + net::{TcpListener, TcpStream}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU8, Ordering}, + }, + thread, + time::{Duration, Instant}, + }; + + use certs::{CaCert, DEFAULT_TRUST_DOMAIN, GridSpiffeClientVerifier, generate_ca, generate_site_cert}; + use rustls::{ + ServerConfig, ServerConnection, StreamOwned, + pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject as _}, + }; + + /// How often the gateway polls each peer. + const POLL: Duration = Duration::from_millis(200); + /// The stated bound on time to exclusion and to rejoin: one poll, plus slack for a loaded runner. + const BOUND: Duration = Duration::from_millis(3 * 200 + 1_000); + /// The load window, short so a partition outlasts it within the test. + const WINDOW: Duration = Duration::from_millis(1_000); + /// The gap between client requests. + const PACE: Duration = Duration::from_millis(10); + /// How long setup may take. + const DEADLINE: Duration = Duration::from_secs(20); + + const SITES: [&str; 3] = ["site-a", "site-b", "site-d"]; + + /// What a fake peer operator publishes. + const READY: u8 = 0; + const NOT_READY: u8 = 1; + /// Load only, as an operator that does not publish readiness. + const OMITTED: u8 = 2; + /// Accept and drop every poll, as a partitioned link. + const PARTITIONED: u8 = 3; + + /// A peer operator's signals endpoint over mutual TLS, publishing per `mode`. + struct Peer { + addr: String, + mode: Arc, + stop: Arc, + } + + impl Peer { + fn start(ca: &CaCert, site: &'static str) -> Self { + let identity = generate_site_cert(ca, site).expect("peer cert"); + let chain = CertificateDer::pem_slice_iter(identity.cert_pem.as_bytes()) + .collect::, _>>() + .expect("peer chain"); + let key = PrivateKeyDer::from_pem_slice(identity.key_pem.as_bytes()).expect("peer key"); + let provider = rustls::crypto::ring::default_provider(); + let verifier = GridSpiffeClientVerifier::new( + ca.cert_pem.as_bytes(), + DEFAULT_TRUST_DOMAIN, + provider.signature_verification_algorithms, + ) + .expect("client verifier"); + let config = Arc::new( + ServerConfig::builder_with_provider(Arc::new(provider)) + .with_safe_default_protocol_versions() + .expect("versions") + .with_client_cert_verifier(verifier) + .with_single_cert(chain, key) + .expect("server config"), + ); + let listener = TcpListener::bind("127.0.0.1:0").expect("bind peer"); + let addr = listener.local_addr().expect("peer addr").to_string(); + let mode = Arc::new(AtomicU8::new(READY)); + let stop = Arc::new(AtomicBool::new(false)); + let (serving, stopping) = (Arc::clone(&mode), Arc::clone(&stop)); + thread::spawn(move || { + for stream in listener.incoming() { + if stopping.load(Ordering::SeqCst) { + return; + } + let Ok(stream) = stream else { continue }; + let publishing = serving.load(Ordering::SeqCst); + if publishing == PARTITIONED { + continue; + } + let _bounded = stream.set_read_timeout(Some(Duration::from_secs(1))); + let Ok(conn) = ServerConnection::new(Arc::clone(&config)) else { + continue; + }; + let mut tls = StreamOwned::new(conn, stream); + let mut request = Vec::new(); + let mut buf = [0_u8; 1024]; + while !request.windows(4).any(|w| w == b"\r\n\r\n") { + match tls.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => request.extend_from_slice(&buf[..n]), + } + } + let body = signals(site, publishing); + // Samples stamped at the Date header read as age zero. + let response = format!( + "HTTP/1.1 200 OK\r\nDate: Thu, 01 Jan 1970 00:00:01 GMT\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _sent = tls.write_all(response.as_bytes()).and_then(|()| tls.flush()); + } + }); + Self { addr, mode, stop } + } + } + + impl Drop for Peer { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + let _woken = TcpStream::connect(&self.addr); + } + } + + /// What `site`'s operator publishes in `mode`: an idle queue, and its readiness verdict. + fn signals(site: &str, mode: u8) -> String { + let labels = format!(r#"grid_site="{site}",grid_provider="pool-{site}""#); + let load = format!("llm_d_epp_average_queue_size{{{labels}}} 0 1000\n"); + match mode { + OMITTED => load, + _ => format!( + "{load}grid_provider_ready{{{labels}}} {} 1000\n", + u8::from(mode == READY) + ), + } + } + + /// A plain HTTP backend answering 200 with its site name. + fn backend(site: &'static str) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind backend"); + let port = listener.local_addr().expect("backend addr").port(); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + thread::spawn(move || { + let mut head = Vec::new(); + let mut buf = [0_u8; 1024]; + while !head.windows(4).any(|w| w == b"\r\n\r\n") { + match stream.read(&mut buf) { + Ok(0) => return, + Ok(n) => head.extend_from_slice(&buf[..n]), + Err(e) if e.kind() == ErrorKind::Interrupted => {}, + Err(_) => return, + } + } + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{site}", + site.len() + ); + let _sent = stream.write_all(response.as_bytes()); + }); + } + }); + port + } + + fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind") + .local_addr() + .expect("addr") + .port() + } + + fn drain(pipe: impl std::io::Read + Send + 'static, lines: Arc>>) { + thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + lines.lock().expect("output").push(line); + } + }); + } + + struct Gateway { + child: Child, + output: Arc>>, + listen: u16, + admin: u16, + } + + impl Gateway { + fn start(work: &Path, backends: &[(&str, u16)], peers: &[(&str, &Peer)], identity: &Path) -> Self { + let (listen, admin) = (free_port(), free_port()); + let clusters = |indent: &str| -> String { + backends + .iter() + .map(|(site, port)| { + format!("{indent}- name: \"pool-{site}\"\n{indent} endpoints: [\"127.0.0.1:{port}\"]\n") + }) + .collect() + }; + let config = work.join("praxis.yaml"); + std::fs::write( + &config, + format!( + "admin:\n address: \"127.0.0.1:{admin}\"\n# The test backends are on loopback.\ninsecure_options:\n allow_private_endpoints: true\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nclusters:\n{top}filter_chains:\n - name: main\n filters:\n - filter: grid_site_route\n model_header: X-Gateway-Model-Name\n - filter: load_balancer\n clusters:\n{inline}", + top = clusters(" "), + inline = clusters(" "), + ), + ) + .expect("praxis config"); + let path = |key: &str| identity.join(key).to_string_lossy().into_owned(); + let candidates: Vec = SITES + .iter() + .map(|site| { + format!(r#"{{"kind":"inference_model","name":"llama","site":"{site}","cluster":"pool-{site}"}}"#) + }) + .collect(); + let peers: Vec = peers + .iter() + .map(|(site, peer)| { + format!( + r#"{{"site":"{site}","addr":"{addr}","server_name":"{site}.grid.internal","authority":"{site}.grid.internal","interval_ms":{poll},"connect_timeout_ms":1000,"request_timeout_ms":1000,"grid_ca_path":"{ca}","client_cert_path":"{cert}","client_key_path":"{key}"}}"#, + addr = peer.addr, + poll = POLL.as_millis(), + ca = path("ca.crt"), + cert = path("tls.crt"), + key = path("tls.key"), + ) + }) + .collect(); + let serving = work.join("serving-config.json"); + std::fs::write( + &serving, + format!( + r#"{{"local_site":"hub","window_secs":60,"load_window_ms":{window},"candidates":[{}],"peers":[{}]}}"#, + candidates.join(","), + peers.join(","), + window = WINDOW.as_millis(), + ), + ) + .expect("serving config"); + let mut child = Command::new(env!("CARGO_BIN_EXE_grid-gateway")) + .arg("--config") + .arg(&config) + .env("GRID_SERVING_CONFIG", &serving) + .env("RUST_LOG", "info") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn gateway"); + let output = Arc::new(Mutex::new(Vec::new())); + drain(child.stdout.take().expect("stdout"), Arc::clone(&output)); + drain(child.stderr.take().expect("stderr"), Arc::clone(&output)); + Self { + child, + output, + listen, + admin, + } + } + + /// One request for `llama`: its status, the site that served it, and Retry-After. + /// The admin listener's Prometheus exposition. + fn metrics(&self) -> String { + let mut stream = TcpStream::connect(("127.0.0.1", self.admin)).expect("connect admin"); + write!( + stream, + "GET /metrics HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("send"); + let mut response = String::new(); + let _read = stream.read_to_string(&mut response); + response + } + + fn request(&self) -> Option<(u16, String, Option)> { + let (head, body) = self.exchange()?; + let status = head.split(' ').nth(1)?.parse().ok()?; + let retry_after = head + .lines() + .find_map(|l| { + l.strip_prefix("retry-after: ") + .or_else(|| l.strip_prefix("Retry-After: ")) + }) + .map(str::to_owned); + Some((status, body, retry_after)) + } + + /// One request for llama: the response head and body. + fn exchange(&self) -> Option<(String, String)> { + let mut stream = TcpStream::connect(("127.0.0.1", self.listen)).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(5))).ok()?; + stream + .write_all( + b"GET /v1/models HTTP/1.1\r\nHost: gw\r\nX-Gateway-Model-Name: llama\r\nConnection: close\r\n\r\n", + ) + .ok()?; + let mut response = String::new(); + stream.read_to_string(&mut response).ok()?; + let (head, body) = response.split_once("\r\n\r\n")?; + Some((head.to_owned(), body.to_owned())) + } + + fn eventually(&self, what: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + DEADLINE; + while !done() { + if Instant::now() > deadline { + let output = self.output.lock().expect("output").join("\n"); + panic!("timed out waiting for {what}; gateway output:\n{output}"); + } + thread::sleep(Duration::from_millis(50)); + } + } + } + + impl Drop for Gateway { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + + /// Requests per serving site over `n` paced requests; any non-200 counts under `error`. + fn split(gateway: &Gateway, n: usize) -> BTreeMap { + let mut counts = BTreeMap::new(); + for _ in 0..n { + let key = match gateway.request() { + Some((200, site, _)) => site, + _ => "error".to_owned(), + }; + *counts.entry(key).or_default() += 1; + thread::sleep(PACE); + } + counts + } + + /// Paced requests until twenty in a row are served by a site `pred` accepts, and how long + /// until the first of them. + /// + /// Twenty, since sites are drawn at random: with one rejected site of three still drawn, + /// twenty accepted in a row has probability (2/3)^20, below 1 in 3000. + fn time_until(gateway: &Gateway, mut pred: impl FnMut(&str) -> bool) -> Duration { + let start = Instant::now(); + let mut streak = 0; + let mut streak_start = start; + while streak < 20 { + assert!(start.elapsed() < DEADLINE, "never settled"); + let sent = Instant::now(); + match gateway.request() { + Some((200, site, _)) if pred(&site) => { + if streak == 0 { + streak_start = sent; + } + streak += 1; + }, + _ => streak = 0, + } + thread::sleep(PACE); + } + streak_start.duration_since(start) + } + + #[test] + fn site_b_not_ready_routes_to_the_other_sites_and_rejoins_when_ready() { + let work = + PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("provider-readiness-{}", std::process::id())); + let _cleared = std::fs::remove_dir_all(&work); + let identity = work.join("identity"); + std::fs::create_dir_all(&identity).expect("identity dir"); + let ca = generate_ca("grid-ca").expect("ca"); + let hub = generate_site_cert(&ca, "hub").expect("hub cert"); + for (key, pem) in [ + ("ca.crt", &ca.cert_pem), + ("tls.crt", &hub.cert_pem), + ("tls.key", &hub.key_pem), + ] { + std::fs::write(identity.join(key), pem).expect("identity file"); + } + let peers: Vec<(&str, Peer)> = SITES.iter().map(|site| (*site, Peer::start(&ca, site))).collect(); + let backends: Vec<(&str, u16)> = SITES.iter().map(|site| (*site, backend(site))).collect(); + let gateway = Gateway::start( + &work, + &backends, + &peers.iter().map(|(site, peer)| (*site, peer)).collect::>(), + &identity, + ); + let site_b = &peers[1].1; + + gateway.eventually("every site serving", || { + let counts = split(&gateway, 9); + SITES.iter().all(|site| counts.contains_key(*site)) + }); + // site-b's provider goes not ready. + site_b.mode.store(NOT_READY, Ordering::SeqCst); + let to_exclusion = time_until(&gateway, |site| site != "site-b"); + assert!( + to_exclusion <= BOUND, + "excluded after {to_exclusion:?}, bound {BOUND:?}" + ); + let excluded = split(&gateway, 200); + assert_eq!( + excluded.get("error"), + None, + "no client errors once excluded: {excluded:?}" + ); + assert_eq!(excluded.get("site-b"), None, "nothing to site-b: {excluded:?}"); + assert!( + // Each takes about 100 of 200 at random, standard deviation about 7. + excluded.get("site-a").copied().unwrap_or(0) >= 70 && excluded.get("site-d").copied().unwrap_or(0) >= 70, + "site-a and site-d share the load: {excluded:?}" + ); + + // Its link partitions past the load window: the last verdict still stands. + site_b.mode.store(PARTITIONED, Ordering::SeqCst); + thread::sleep(WINDOW * 3); + let partitioned = split(&gateway, 50); + assert_eq!( + partitioned.get("site-b"), + None, + "a silent peer is not readmitted: {partitioned:?}" + ); + assert_eq!(partitioned.get("error"), None, "{partitioned:?}"); + + // It heals with an operator that no longer publishes readiness: absent reads as ready. + site_b.mode.store(OMITTED, Ordering::SeqCst); + let healed = Instant::now(); + while !gateway + .request() + .is_some_and(|(status, site, _)| status == 200 && site == "site-b") + { + assert!(healed.elapsed() < DEADLINE, "site-b never rejoined without the series"); + thread::sleep(PACE); + } + assert!( + healed.elapsed() <= BOUND, + "rejoined without the series after {:?}", + healed.elapsed() + ); + + // Back to publishing, not ready, then ready. + site_b.mode.store(NOT_READY, Ordering::SeqCst); + time_until(&gateway, |site| site != "site-b"); + site_b.mode.store(READY, Ordering::SeqCst); + let start = Instant::now(); + while !gateway + .request() + .is_some_and(|(status, site, _)| status == 200 && site == "site-b") + { + assert!(start.elapsed() < DEADLINE, "site-b never rejoined"); + thread::sleep(PACE); + } + let to_rejoin = start.elapsed(); + assert!(to_rejoin <= BOUND, "rejoined after {to_rejoin:?}, bound {BOUND:?}"); + let rejoined = split(&gateway, 300); + assert_eq!(rejoined.get("error"), None, "{rejoined:?}"); + // Each takes about 100 of 300 at random, standard deviation about 8. + for site in SITES { + let share = rejoined.get(site).copied().unwrap_or(0); + assert!( + (70..=130).contains(&share), + "{site} took {share} of 300, a herd: {rejoined:?}" + ); + } + + // Every site not ready: a known model answers 503 with Retry-After, not 404. + for (_, peer) in &peers { + peer.mode.store(NOT_READY, Ordering::SeqCst); + } + gateway.eventually("503 once every site is excluded", || { + gateway.request().is_some_and(|(status, _, after)| { + status == 503 + && after + .and_then(|a| a.parse::().ok()) + .is_some_and(|a| (3..=7).contains(&a)) + }) + }); + + let metrics = gateway.metrics(); + let counted = |prefix: &str| -> f64 { + metrics + .lines() + .filter(|line| line.starts_with(prefix)) + .filter_map(|line| line.rsplit_once(' ').and_then(|(_, value)| value.parse::().ok())) + .sum() + }; + for site in SITES { + let routed = counted(&format!("grid_route_decisions_total{{site=\"{site}\",")); + assert!(routed > 0.0, "no routing decisions for {site}:\n{metrics}"); + } + let refused = counted("grid_route_decisions_total{site=\"\",reason=\"not_ready\"}"); + assert!(refused > 0.0, "the 503s are not counted:\n{metrics}"); + let score = "grid_route_site_score{site=\"site-b\",cluster=\"pool-site-b\"} NaN"; + assert!(metrics.contains(score), "an excluded site keeps a score:\n{metrics}"); + + drop(gateway); + let _cleaned = std::fs::remove_dir_all(&work); + } +} diff --git a/gateway/tests/unrouted.rs b/gateway/tests/unrouted.rs new file mode 100644 index 00000000..5b3f3320 --- /dev/null +++ b/gateway/tests/unrouted.rs @@ -0,0 +1,218 @@ +//! A request the grid router cannot route is answered by the router and logged below WARN. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::expect_used, + clippy::panic, + clippy::disallowed_methods, + clippy::arithmetic_side_effects, + clippy::too_many_lines, + reason = "tests; waits poll a deadline with thread::sleep" +)] +mod tests { + use std::{ + io::{BufRead as _, BufReader, Read as _, Write as _}, + net::{TcpListener, TcpStream}, + path::PathBuf, + process::{Child, Command, Stdio}, + sync::{Arc, Mutex}, + thread, + time::{Duration, Instant}, + }; + + /// How long the gateway may take to start listening. + const DEADLINE: Duration = Duration::from_secs(20); + + fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind") + .local_addr() + .expect("addr") + .port() + } + + /// Collect `pipe`'s lines into `lines` on a background thread. + fn drain(pipe: impl std::io::Read + Send + 'static, lines: Arc>>) { + thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + lines.lock().expect("output").push(line); + } + }); + } + + /// The gateway with its output collected in the background. + struct Gateway { + child: Child, + output: Arc>>, + listen: u16, + admin: u16, + } + + impl Gateway { + /// Route `llama` to a local cluster whose only endpoint refuses connections. + fn start(work: &std::path::Path) -> Self { + std::fs::create_dir_all(work).expect("work dir"); + let (listen, admin, dead) = (free_port(), free_port(), free_port()); + let serving = work.join("serving-config.json"); + std::fs::write( + &serving, + r#"{"local_site":"local","window_secs":60,"load_window_ms":30000,"candidates":[{"kind":"inference_model","name":"llama","site":"local","cluster":"pool-local"}],"peers":[]}"#, + ) + .expect("serving config"); + let config = work.join("praxis.yaml"); + std::fs::write( + &config, + format!( + "insecure_options:\n allow_private_endpoints: true\nadmin:\n address: \"127.0.0.1:{admin}\"\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters:\n - filter: grid_site_route\n model_header: X-Model\n - filter: load_balancer\n clusters:\n - name: pool-local\n endpoints: [\"127.0.0.1:{dead}\"]\n" + ), + ) + .expect("praxis config"); + let mut child = Command::new(env!("CARGO_BIN_EXE_grid-gateway")) + .arg("--config") + .arg(&config) + .env("GRID_SERVING_CONFIG", &serving) + .env("RUST_LOG", "debug") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn gateway"); + let output = Arc::new(Mutex::new(Vec::new())); + drain(child.stdout.take().expect("stdout"), Arc::clone(&output)); + drain(child.stderr.take().expect("stderr"), Arc::clone(&output)); + let mut gateway = Self { + child, + output, + listen, + admin, + }; + let deadline = Instant::now() + DEADLINE; + while TcpStream::connect(("127.0.0.1", listen)).is_err() { + assert!( + Instant::now() < deadline && gateway.child.try_wait().ok().flatten().is_none(), + "gateway never listened; output:\n{}", + gateway.output() + ); + thread::sleep(Duration::from_millis(50)); + } + gateway + } + + fn output(&self) -> String { + self.output.lock().expect("output").join("\n") + } + + fn lines(&self) -> usize { + self.output.lock().expect("output").len() + } + + /// The admin listener's Prometheus exposition. + fn metrics(&self) -> String { + let mut stream = TcpStream::connect(("127.0.0.1", self.admin)).expect("connect admin"); + write!( + stream, + "GET /metrics HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("send"); + let mut response = String::new(); + let _read = stream.read_to_string(&mut response); + response + } + + /// The status code of a GET for `path` with `headers`. + fn status(&self, path: &str, headers: &str) -> u16 { + let mut stream = TcpStream::connect(("127.0.0.1", self.listen)).expect("connect"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("read timeout"); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: grid.example\r\n{headers}Connection: close\r\n\r\n" + ) + .expect("send"); + let mut response = String::new(); + let _read = stream.read_to_string(&mut response); + response + .split_whitespace() + .nth(1) + .and_then(|code| code.parse().ok()) + .unwrap_or_else(|| panic!("no status line in {response:?}")) + } + } + + impl Drop for Gateway { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + + /// Whether a log line is at WARN or ERROR. + fn loud(line: &str) -> bool { + line.split_whitespace() + .take(3) + .any(|word| word.contains("WARN") || word.contains("ERROR")) + } + + #[test] + fn an_unrouted_request_is_answered_by_the_router_below_warn() { + let work = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("unrouted-{}", std::process::id())); + let _cleared = std::fs::remove_dir_all(&work); + let gateway = Gateway::start(&work); + let before = gateway.lines(); + + assert_eq!(gateway.status("/health", ""), 400, "no model in the request"); + assert_eq!( + gateway.status("/v1/models", "X-Model: granite\r\n"), + 404, + "a model with no admitted candidate" + ); + + let deadline = Instant::now() + Duration::from_secs(5); + while !gateway.output().contains("grid_site_route: no model in the request") { + assert!( + Instant::now() < deadline, + "no debug line; output:\n{}", + gateway.output() + ); + thread::sleep(Duration::from_millis(50)); + } + let metrics = gateway.metrics(); + // Unmeasured, so the order ranks it last with an infinite score. + let score = "grid_route_site_score{site=\"local\",cluster=\"pool-local\"} inf"; + assert!(metrics.contains(score), "no {score:?} in:\n{metrics}"); + // No model and an unknown model both count as bad_request. + let refused = "grid_route_decisions_total{site=\"\",reason=\"bad_request\"} 2"; + assert!(metrics.contains(refused), "no {refused:?} in:\n{metrics}"); + // Labels come from the serving config and a closed set, never from the request. + for series in metrics + .lines() + .filter(|line| line.starts_with("grid_route_decisions_total{")) + { + let label = |name: &str| { + series + .split_once(&format!("{name}=\"")) + .and_then(|(_, rest)| rest.split_once('"')) + .map_or_else(|| panic!("no {name} in {series}"), |(value, _)| value.to_owned()) + }; + assert!( + ["", "local"].contains(&label("site").as_str()), + "request-derived site: {series}" + ); + assert!( + ["routed", "fallback", "not_ready", "no_route", "bad_request", "shed"] + .contains(&label("reason").as_str()), + "reason outside the closed set: {series}" + ); + } + let output = gateway.output.lock().expect("output").clone(); + let loud_lines: Vec<&String> = output.iter().skip(before).filter(|line| loud(line)).collect(); + assert!( + loud_lines.is_empty(), + "an unrouted request logged at WARN or ERROR: {loud_lines:#?}" + ); + + drop(gateway); + let _cleaned = std::fs::remove_dir_all(&work); + } +} diff --git a/signals/src/signals.rs b/signals/src/signals.rs index 1590913a..d156ace6 100644 --- a/signals/src/signals.rs +++ b/signals/src/signals.rs @@ -154,17 +154,26 @@ impl LoadStore { format!("{site}/{cluster}").into_boxed_str() } - /// Most recent sample of `metric` for `key`. Test-only since scoring reads - /// [`Self::window_worst`]. - #[cfg(test)] + /// Most recent sample of `metric` for `key`. + #[must_use] pub fn latest(&self, key: &str, metric: &str) -> Option { let provider = self.providers.get(key)?; provider.metrics.get(metric)?.samples.last().copied() } + /// When anything was last observed for `key`, across all its metrics. + #[must_use] + pub fn newest_at(&self, key: &str) -> Option { + let provider = self.providers.get(key)?; + provider + .metrics + .values() + .filter_map(|series| series.samples.last().map(|sample| sample.at_ms)) + .max() + } + /// Most recent sample of `metric` for `key` younger than `max_age_ms`. - /// Test-only since scoring reads [`Self::window_worst`]. - #[cfg(test)] + #[must_use] pub fn fresh(&self, key: &str, metric: &str, now_ms: i64, max_age_ms: i64) -> Option { // Range starts at zero: a future timestamp (publisher clock ahead) yields // a negative age that would otherwise read as fresh forever. diff --git a/tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml b/tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml new file mode 100644 index 00000000..e89bbe1b --- /dev/null +++ b/tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml @@ -0,0 +1,530 @@ +apiVersion: forge.praxis.dev/v1alpha1 +kind: Environment +metadata: + name: grid-single-cluster-multi-gateway +spec: + runtime: + provider: docker + clusterPrefix: grid-single-cluster-multi-gateway + network: + crossCluster: true + dnsZone: grid-single-cluster-multi-gateway.test + clusters: + - name: single + stacks: + - metallb + - tls-bootstrap + - provider-a-operator-base + - vcr-backend + - provider-a-site + - provider-gateway-a + - provider-gateway-b + - provider-gateway-c + - consumer-gateway-a + - consumer-gateway-b + properties: + region: single + role: combined + siteName: single + gatewayImage: registry.example/ai:run-4 + operatorImage: registry.example/operator:run-4 + vcrImage: registry.example/vcr:run-4 + imagePullPolicy: Never + gatewayImageRepo: registry.example/ai + gatewayImageTag: run-4 + operatorImageRepo: registry.example/operator + operatorImageTag: run-4 + overlaySyncImage: registry.example/sync:run-4 + overlaySyncImageRepo: registry.example/sync + overlaySyncImageTag: run-4 + stacks: + tls-bootstrap: + description: Ephemeral same-CA identities for this isolated qualification + steps: + - type: manifest + path: resources/common/grid-system-namespace.yaml + - type: exec + command: + - bash + - -c + - set -eu; d=$(mktemp -d); trap 'rm -rf "${d}"' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/O=ai-grid/CN=qualification-ca" -keyout "${d}/ca.key" -out "${d}/ca.crt" >/dev/null 2>&1; openssl req -newkey rsa:2048 -nodes -subj "/O=ai-grid/CN=grid-gateway.grid.internal" -keyout "${d}/gateway.key" -out "${d}/gateway.csr" >/dev/null 2>&1; printf "subjectAltName=DNS:grid-gateway.grid.internal,DNS:provider-a.grid.internal,DNS:provider-b.grid.internal,DNS:provider-c.grid.internal,DNS:grid-system.svc.cluster.local" > "${d}/ext.cnf"; openssl x509 -req -days 1 -in "${d}/gateway.csr" -CA "${d}/ca.crt" -CAkey "${d}/ca.key" -CAcreateserial -out "${d}/gateway.crt" -extfile "${d}/ext.cnf" >/dev/null 2>&1; kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create secret generic consumer-gateway-tls --from-file=ca.crt="${d}/ca.crt" --from-file=tls.crt="${d}/gateway.crt" --from-file=tls.key="${d}/gateway.key" --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - >/dev/null; kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create secret generic vcr-inference-credential --from-literal=token="$(openssl rand -hex 32)" --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - >/dev/null + metallb: + description: MetalLB load balancer with auto-configured address pool + steps: + - type: url + url: https://raw.githubusercontent.com/metallb/metallb/v0.14.9/config/manifests/metallb-native.yaml + sha256: 951065e85692aa106f1bb5d5a487d9306154923a794ab1d82122881cbaf588e4 + - type: wait + resource: deployment/controller + namespace: metallb-system + condition: available + timeout: 120s + - type: metallb-auto-pool + name: forge-pool + provider-a-operator-base: + description: Single Grid operator for the shared Kind control plane + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: 0.1.0 + namespace: grid-system + values: + image: + repository: '{{ cluster.properties.operatorImageRepo }}' + tag: '{{ cluster.properties.operatorImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + swim: + siteName: single + seeds: '' + service: + enabled: true + type: LoadBalancer + gateway: + serviceName: provider-gateway-a + port: '8443' + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: 120s + - type: helm + release: grid-operator + chart: charts/grid-operator + version: 0.1.0 + namespace: grid-system + values: + image: + repository: '{{ cluster.properties.operatorImageRepo }}' + tag: '{{ cluster.properties.operatorImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + swim: + siteName: single + seeds: '' + advertiseAddress: 172.18.255.231:7946 + service: + enabled: true + type: LoadBalancer + gateway: + serviceName: provider-gateway-a + port: '8443' + vcr-backend: + description: vllm-vcr inference backend for demo scenarios + steps: + - type: manifest + path: resources/common/grid-system-namespace.yaml + - type: template-manifest + path: resources/common/vcr-provider-a.yaml + - type: template-manifest + path: resources/common/vcr-provider-b.yaml + - type: template-manifest + path: resources/common/vcr-provider-c.yaml + - type: manifest + path: resources/common/backend-network-policy.yaml + - type: manifest + path: resources/common/qualification-client.yaml + - type: wait + resource: deployment/vcr-inference-provider-a + namespace: grid-system + condition: available + timeout: 120s + provider-a-site: + description: Single GridSite with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site-a + chart: charts/grid-site + version: 0.1.0 + namespace: grid-system + values: + commonLabels: + grid.praxis.fast/auto-discover-sites: 'true' + gridNetwork: + name: grid-single-cluster-multi-gateway + gridId: grid-single-cluster-multi-gateway-v1 + region: provider-a + zone: provider-a-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + swim: + probeInterval: 5s + suspicionTimeout: 15s + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: consumer-gateway-a + namespace: grid-system + localSiteName: single + - name: consumer-gateway-b + namespace: grid-system + localSiteName: single + gridSite: + name: single + region: single + zone: single-1 + providerSiteLabel: single + inferenceProviders: + - name: vcr-provider-a-provider + gridNetworkRef: grid-single-cluster-multi-gateway + gatewayRef: provider-gateway-a + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-provider-a.grid-system.svc.cluster.local:8000 + siteSelector: + matchLabels: + grid.praxis.fast/provider-site: single + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: 10s + timeout: 5s + - name: vcr-provider-b-provider + gridNetworkRef: grid-single-cluster-multi-gateway + gatewayRef: provider-gateway-a + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-provider-b.grid-system.svc.cluster.local:8000 + siteSelector: + matchLabels: {} + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: 10s + timeout: 5s + - name: vcr-provider-c-provider + gridNetworkRef: grid-single-cluster-multi-gateway + gatewayRef: provider-gateway-b + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-provider-c.grid-system.svc.cluster.local:8000 + siteSelector: + matchLabels: {} + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: 10s + timeout: 5s + provider-gateway-a: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: template-file + source: configs/provider/praxis-a.yaml + target: .forge/runtime/{{ cluster.name }}/provider-a-praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create configmap provider-gateway-a-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider-a-praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - + - type: helm + release: provider-gateway-a + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: provider-gateway-a + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: provider-gateway-a-config + port: + containerPort: 8443 + name: https-mtls + service: + type: LoadBalancer + port: 8443 + health: + readiness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: consumer-gateway-tls + credentials: + - name: vcr-inference-credential + mountPath: /etc/praxis/credentials/vcr-inference + podLabels: + grid.praxis.fast/backend-access: provider-gateway + grid.praxis.fast/provider-site: '{{ cluster.name }}' + - type: wait + resource: deployment/provider-gateway-a + namespace: grid-system + condition: available + timeout: 120s + provider-gateway-b: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: template-file + source: configs/provider/praxis-b.yaml + target: .forge/runtime/{{ cluster.name }}/provider-b-praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create configmap provider-gateway-b-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider-b-praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - + - type: helm + release: provider-gateway-b + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: provider-gateway-b + replicaCount: 1 + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: provider-gateway-b-config + port: + containerPort: 8443 + name: https-mtls + service: + type: LoadBalancer + port: 8443 + health: + readiness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: consumer-gateway-tls + credentials: + - name: vcr-inference-credential + mountPath: /etc/praxis/credentials/vcr-inference + podLabels: + grid.praxis.fast/backend-access: provider-gateway + grid.praxis.fast/provider-site: '{{ cluster.name }}' + - type: wait + resource: deployment/provider-gateway-b + namespace: grid-system + condition: available + timeout: 120s + provider-gateway-c: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: template-file + source: configs/provider/praxis-c.yaml + target: .forge/runtime/{{ cluster.name }}/provider-c-praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create configmap provider-gateway-c-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider-c-praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - + - type: helm + release: provider-gateway-c + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: provider-gateway-c + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: provider-gateway-c-config + port: + containerPort: 8443 + name: https-mtls + service: + type: LoadBalancer + port: 8443 + health: + readiness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: consumer-gateway-tls + credentials: + - name: vcr-inference-credential + mountPath: /etc/praxis/credentials/vcr-inference + podLabels: + grid.praxis.fast/backend-access: provider-gateway + grid.praxis.fast/provider-site: '{{ cluster.name }}' + - type: wait + resource: deployment/provider-gateway-c + namespace: grid-system + condition: available + timeout: 120s + consumer-gateway-a: + description: Praxis consumer gateway with operator-managed overlay + steps: + - type: template-file + source: configs/consumer/praxis-a.yaml + target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} -n grid-system create configmap consumer-gateway-a-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway-a + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: consumer-gateway-a + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: consumer-gateway-a-config + service: + type: ClusterIP + overlay: + enabled: true + existingConfigMap: grid-overlay-grid-single-cluster--consumer-gateway-a-db750bac + tls: + enabled: true + existingSecret: consumer-gateway-tls + podLabels: + grid.praxis.fast/consumer-site: '{{ cluster.name }}' + - type: wait + resource: deployment/consumer-gateway-a + namespace: grid-system + condition: available + timeout: 120s + consumer-gateway-b: + description: Praxis consumer gateway with operator-managed overlay + steps: + - type: template-file + source: configs/consumer/praxis-b.yaml + target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} -n grid-system create configmap consumer-gateway-b-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway-b + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: consumer-gateway-b + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: consumer-gateway-b-config + service: + type: ClusterIP + overlay: + enabled: true + existingConfigMap: grid-overlay-grid-single-cluster--consumer-gateway-b-de751065 + tls: + enabled: true + existingSecret: consumer-gateway-tls + podLabels: + grid.praxis.fast/consumer-site: '{{ cluster.name }}' + - type: wait + resource: deployment/consumer-gateway-b + namespace: grid-system + condition: available + timeout: 120s