diff --git a/charts/grid-operator/README.md b/charts/grid-operator/README.md index e5cea1a3e..1a5be7d51 100644 --- a/charts/grid-operator/README.md +++ b/charts/grid-operator/README.md @@ -156,6 +156,7 @@ RELEASE=grid-operator; NAMESPACE=grid-system; for crd in agenttoolproviders grid |-----|------|---------|-------------| | `crds.enabled` | bool | `true` | Install and upgrade the Grid CRDs. `false` when a platform owns them. | | `crds.keep` | bool | `true` | Keep the CRDs on `helm uninstall` and an Argo CD delete or prune. | +| `grid.providers` | object | `{}` | InferenceProviders this site serves, keyed by name. `model` defaults to the name, `providerKind` to `vllm`, `backendKind` to `local_model`. `maxRunning` is the most requests one endpoint runs at once (vLLM max-num-seqs); without it the provider's capacity, and so its saturation, is unknown. | | `rbac.enrollmentNamespace` | string | `""` | The grid-enrollment namespace. The render fails if the operator would get Secret access there. | | `rbac.metricsScraper` | bool | `true` | Create the metrics scraper ServiceAccount, allowed only GET on the nonResourceURL /metrics, and let the operator mint short-lived tokens for it. An llm-d EPP serving bearer-authenticated metrics (the default) admits a scrape with that token (metricsConfig.auth type serviceAccountToken). The operator never sends its own token. | | `replicaCount` | int | `1` | Operator replicas. Must be 1 (schema-enforced). | diff --git a/charts/grid-operator/templates/crds/inferenceprovider.yaml b/charts/grid-operator/templates/crds/inferenceprovider.yaml index 57e33f9ab..eeb1216ea 100644 --- a/charts/grid-operator/templates/crds/inferenceprovider.yaml +++ b/charts/grid-operator/templates/crds/inferenceprovider.yaml @@ -24,8 +24,19 @@ spec: - jsonPath: .spec.providerKind name: Provider type: string + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + priority: 1 + type: string - jsonPath: .status.phase name: Phase + priority: 1 type: string name: v1alpha1 schema: @@ -289,6 +300,17 @@ spec: - message: set exactly one of caSecretRef and caConfigMapRef rule: has(self.caSecretRef) != has(self.caConfigMapRef) type: object + maxRunning: + description: |- + Most requests one endpoint runs at once: vLLM max-num-seqs. + + The site's operator multiplies it by the EPP's fresh ready endpoints and publishes + the product as `grid_provider_capacity_requests`, the capacity gateways weigh the + provider's load against. When absent, or no endpoint is fresh, capacity is unknown. + format: uint32 + minimum: 1.0 + nullable: true + type: integer metricsConfig: description: |- Prometheus metrics scraping configuration. @@ -407,6 +429,14 @@ spec: description: Metric name for normalised queue depth (0.0–1.0). nullable: true type: string + readyEndpoints: + description: |- + Metric name counting the pool's ready endpoints, read for the `Ready` condition. + + Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`. + Filtered by `poolName` when set. Zero marks the provider not ready. + nullable: true + type: string type: object staleMetricsSeconds: description: |- @@ -764,6 +794,46 @@ spec: description: Observed status of an [`InferenceProvider`]. nullable: true properties: + conditions: + description: |- + Observed conditions. `Ready` says whether the provider can currently serve a request. + + Written by the operator's signals loop, never by provider reconciliation. + items: + description: One observed condition, shaped like `metav1.Condition`. + properties: + lastTransitionTime: + description: When `status` last changed, RFC 3339. + format: date-time + type: string + message: + default: "" + description: Human-readable detail. + type: string + observedGeneration: + description: The `metadata.generation` this was computed against. + format: int64 + nullable: true + type: integer + reason: + description: CamelCase reason for the status. + type: string + status: + description: '`True`, `False`, or `Unknown`.' + type: string + type: + description: Condition type, such as `Ready`. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map matchingSites: default: [] description: Sites matched by the site selector. @@ -811,6 +881,17 @@ spec: `HealthCheckTlsIdentityMismatch` nullable: true type: string + state: + description: |- + `Ready`, `NotReady`, or `Unknown`: the `Ready` condition's status, for display. + + Written with the condition, never by provider reconciliation. + enum: + - Ready + - NotReady + - Unknown + nullable: true + type: string type: object required: - spec diff --git a/charts/grid-operator/templates/deployment.yaml b/charts/grid-operator/templates/deployment.yaml index 5968ff6c1..6524fcd84 100644 --- a/charts/grid-operator/templates/deployment.yaml +++ b/charts/grid-operator/templates/deployment.yaml @@ -108,15 +108,6 @@ spec: value: {{ printf "%s-metrics-scraper" (include "grid-operator.fullname" .) | quote }} - name: GRID_METRICS_SCRAPER_NAMESPACE value: {{ .Release.Namespace | quote }} - # Binds each scrape token to this pod, so it stops working when the pod goes. - - name: GRID_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: GRID_POD_UID - valueFrom: - fieldRef: - fieldPath: metadata.uid {{- end }} {{- if (.Values.signals).enabled }} - name: GRID_SIGNALS_LOCAL_ADDR diff --git a/charts/grid-operator/templates/inferenceprovider.yaml b/charts/grid-operator/templates/inferenceprovider.yaml index 576efd329..d2ed256c1 100644 --- a/charts/grid-operator/templates/inferenceprovider.yaml +++ b/charts/grid-operator/templates/inferenceprovider.yaml @@ -23,6 +23,9 @@ spec: {{- with .capacityWeight }} capacityWeight: {{ . }} {{- end }} + {{- with .maxRunning }} + maxRunning: {{ . }} + {{- end }} {{- with .routingClusterRef }} routingClusterRef: {{ . | quote }} {{- end }} diff --git a/charts/grid-operator/tests/grid_test.yaml b/charts/grid-operator/tests/grid_test.yaml index 41e1f03b0..3d73b8b1f 100644 --- a/charts/grid-operator/tests/grid_test.yaml +++ b/charts/grid-operator/tests/grid_test.yaml @@ -135,6 +135,32 @@ tests: - name: Qwen3-Coder-30B-A3B capabilities: [text_generation] + - it: passes maxRunning through to the provider spec + template: templates/inferenceprovider.yaml + set: + site.name: east + grid.id: lab + grid.providers: + qwen3: + endpoint: http://qwen3.llm-d.svc:80 + maxRunning: 64 + asserts: + - equal: + path: spec.maxRunning + value: 64 + + - it: leaves maxRunning unset when a provider omits it + template: templates/inferenceprovider.yaml + set: + site.name: east + grid.id: lab + grid.providers: + qwen3: + endpoint: http://qwen3.llm-d.svc:80 + asserts: + - notExists: + path: spec.maxRunning + - it: keeps YAML-ambiguous names strings templates: - templates/gridnetwork.yaml diff --git a/charts/grid-operator/values.yaml b/charts/grid-operator/values.yaml index 0938d7f2e..927f775fd 100644 --- a/charts/grid-operator/values.yaml +++ b/charts/grid-operator/values.yaml @@ -28,11 +28,14 @@ grid: swimKeySecretName: grid-swim-key # -- InferenceProviders this site serves, keyed by name, as in the grid-site chart. # model defaults to the name, providerKind to vllm, and backendKind to local_model. + # maxRunning is the most requests one endpoint runs at once (vLLM max-num-seqs); without + # it the provider's capacity, and so its saturation, is unknown. providers: {} # qwen3: # model: Qwen3-Coder-30B-A3B # endpoint: http://qwen3-epp-service.llm-d.svc:80 # routingClusterRef: site-d + # maxRunning: 64 # -- Grid CRDs. crds: diff --git a/charts/praxis-gateway/README.md b/charts/praxis-gateway/README.md index f86c8f822..163a48434 100644 --- a/charts/praxis-gateway/README.md +++ b/charts/praxis-gateway/README.md @@ -228,6 +228,10 @@ Praxis AI image; these values may advance independently. | `gatewayConfig.auth.validateCA` | object | empty | CA for the validate call (`configMap` or `secret`, `key`). Set as `SSL_CERT_FILE`, which replaces the platform trust store for the validate call and https backends without a per-backend CA or `upstreamCA`. mutual_tls backends and `upstreamCA` are unaffected. See the recipe below. | | `networkPolicy.enabled` | bool | `false` | Render a NetworkPolicy that limits which pods can reach the listener port, where the CNI enforces NetworkPolicy. It is not authentication. Node and host-network traffic handling is CNI-specific (OVN-Kubernetes: the `policy-group.network.openshift.io/host-network` label), and a LoadBalancer with `externalTrafficPolicy: Cluster` can SNAT clients to node IPs. | | `networkPolicy.from` | list | `[]` | NetworkPolicyPeer entries allowed in. Required when enabled. With `auth.mode: none`, list only the authenticating front. `{podSelector: {}}` admits every pod in this namespace. An empty `namespaceSelector` and an `ipBlock` of `0.0.0.0/0` or `::/0` admit everyone and fail the render. An all-address `ipBlock` with `except` entries is allowed. The check reads selector emptiness and the cidr only, so `matchExpressions` that happen to select every pod pass. A provider gateway behind a LoadBalancer that SNATs clients to node IPs needs `ipBlock` peers for those node addresses. | +| `metricsListener.enabled` | bool | `false` | Serve `GET /metrics` over TLS on its own port and ClusterIP Service, for an in-cluster Prometheus. The admin listener refuses a non-loopback Host, so Prometheus cannot scrape it. Needs the grid-gateway image, `existingSecret`, `fromNamespaces`, and `networkPolicy.enabled`. The port answers only `/metrics` but has no authentication, so the NetworkPolicy is its access control, and that holds only where the CNI enforces NetworkPolicy. | +| `metricsListener.existingSecret` | string | `""` | Secret with `tls.crt` and `tls.key`. On OpenShift, request it with `metricsListener.service.annotations` `service.beta.openshift.io/serving-cert-secret-name`. The listener reloads the cert when the Secret changes. | +| `metricsListener.fromNamespaces` | list | `[]` | Namespace names allowed to reach the metrics port, for example `openshift-user-workload-monitoring`. | +| `metricsListener.serviceMonitor.enabled` | bool | `false` | Render a ServiceMonitor that verifies the cert against `caConfigMap` (for example `openshift-service-ca.crt`, key `service-ca.crt`) and renames Praxis's `cluster` label to `backend`, since ACM uses `cluster` for the managed cluster. | | `gatewayConfig.upstreamCA.secretName` | string | `""` | CA bundle for backend TLS without a per-cluster CA (`upstream_ca_file`). | | `gatewayConfig.listenerTls.enabled` | bool | `false` | Terminate TLS at the listener from `existingSecret`, in render or BYO mode. Names the port `https`. The cert mounts at `listenerTls.mountPath` (`/etc/praxis/listener-tls`), so a BYO config moving off `tls.enabled` must point its listener `cert_path`/`key_path` there. On OpenShift, annotate the Service with `service.beta.openshift.io/serving-cert-secret-name`. | | `port.containerPort` | int | `8080` | Container port. | @@ -274,6 +278,25 @@ Praxis AI image; these values may advance independently. | `topologySpreadConstraints` | list | `[]` | Topology spread constraints. | | `priorityClassName` | string | `""` | Pod priority class. | +## Metrics + +The metrics listener serves the Praxis registry, including the grid gateway's own series. With the ServiceMonitor, Praxis's `cluster` label arrives as `backend`. + +| Metric | Labels | Meaning | +|--------|--------|---------| +| `grid_route_decisions_total` | `site`, `reason` | Requests `grid_site_route` decided. `site` is a site name from the serving config, or empty for a refusal. No label comes from the request. | +| `grid_route_site_score` | `site`, `cluster` | The queue depth the last route order used for each candidate, lower first. `inf` when unmeasured, `NaN` when excluded or demoted, or when the pair left the topology. | + +`reason` is one of five values, and adding one is a deliberate change: + +| `reason` | `site` | Response | +|----------|--------|----------| +| `routed` | chosen site | Sent to a healthy site. How it ranked is in `grid_route_site_score`. | +| `fallback` | chosen site | Sent to a demoted site because no healthy one was left. | +| `not_ready` | empty | 503: every candidate was excluded. | +| `no_route` | empty | 503: an admitted candidate had no route from this gateway. | +| `bad_request` | empty | 400 or 404: no model, or a model no candidate serves. | + ## Security The chart enforces Kubernetes restricted security defaults: diff --git a/charts/praxis-gateway/templates/_helpers.tpl b/charts/praxis-gateway/templates/_helpers.tpl index b09ec3bad..5de33a5f5 100644 --- a/charts/praxis-gateway/templates/_helpers.tpl +++ b/charts/praxis-gateway/templates/_helpers.tpl @@ -555,3 +555,36 @@ RUST_LOG for the gateway and overlay-sync: log.filter when set, else log.level, {{- $log := .Values.log | default dict -}} {{- $log.filter | default $log.level -}} {{- end }} + +{{/* +The metrics listener needs the grid-gateway image, a cert, and the NetworkPolicy that +limits its port; without the policy any pod could scrape it. +*/}} +{{- define "praxis-gateway.validateMetricsListener" -}} +{{- $m := .Values.metricsListener }} +{{- if $m.enabled }} +{{- if ne .Values.image.flavor "grid-gateway" }} +{{- fail "metricsListener needs image.flavor grid-gateway" }} +{{- end }} +{{- if not $m.existingSecret }} +{{- fail "metricsListener.existingSecret is required: the listener serves TLS only" }} +{{- end }} +{{- if not .Values.networkPolicy.enabled }} +{{- fail "metricsListener needs networkPolicy.enabled, which limits the metrics port to metricsListener.fromNamespaces" }} +{{- end }} +{{- if not $m.fromNamespaces }} +{{- fail "metricsListener.fromNamespaces needs at least one namespace" }} +{{- end }} +{{- $taken := list (int .Values.port.containerPort) }} +{{- if and .Values.overlay.enabled .Values.overlay.sidecar.enabled }}{{ $taken = append $taken 9091 }}{{ end }} +{{- if has (int $m.port) $taken }} +{{- fail (printf "metricsListener.port %d collides with another gateway pod port" (int $m.port)) }} +{{- end }} +{{- end }} +{{- if and $m.serviceMonitor.enabled (not $m.enabled) }} +{{- fail "metricsListener.serviceMonitor needs metricsListener.enabled" }} +{{- end }} +{{- if and $m.serviceMonitor.enabled (not $m.serviceMonitor.caConfigMap.name) }} +{{- fail "metricsListener.serviceMonitor.caConfigMap.name is required to verify the metrics cert" }} +{{- end }} +{{- end }} diff --git a/charts/praxis-gateway/templates/deployment.yaml b/charts/praxis-gateway/templates/deployment.yaml index 4952793e6..db0790b3d 100644 --- a/charts/praxis-gateway/templates/deployment.yaml +++ b/charts/praxis-gateway/templates/deployment.yaml @@ -3,6 +3,7 @@ {{- include "praxis-gateway.validateConfig" . }} {{- include "praxis-gateway.validateMounts" . }} {{- include "praxis-gateway.validateProbes" . }} +{{- include "praxis-gateway.validateMetricsListener" . }} apiVersion: apps/v1 kind: Deployment metadata: @@ -192,7 +193,7 @@ spec: {{- $userLog := false }} {{- range .Values.env }}{{- if eq .name "RUST_LOG" }}{{- $userLog = true }}{{- end }}{{- end }} {{- $rustLog := and (not $userLog) (include "praxis-gateway.rustLog" .) }} - {{- if or $rustLog .Values.env (.Values.gridServing).enabled $withCA }} + {{- if or $rustLog .Values.env (.Values.gridServing).enabled $withCA (.Values.metricsListener).enabled }} env: {{- if $rustLog }} - name: RUST_LOG @@ -209,6 +210,16 @@ spec: - name: SSL_CERT_FILE value: {{ printf "%s/%s" $validateCA.mountPath $validateCA.key | quote }} {{- end }} + {{- with .Values.metricsListener }} + {{- if .enabled }} + - name: GRID_METRICS_ADDR + value: {{ printf "0.0.0.0:%d" (int .port) | quote }} + - name: GRID_METRICS_TLS_CERT + value: {{ printf "%s/tls.crt" .mountPath | quote }} + - name: GRID_METRICS_TLS_KEY + value: {{ printf "%s/tls.key" .mountPath | quote }} + {{- end }} + {{- end }} {{- end }} securityContext: runAsNonRoot: true @@ -227,6 +238,11 @@ spec: - name: {{ include "praxis-gateway.portName" . }} containerPort: {{ .Values.port.containerPort }} protocol: {{ .Values.port.protocol | quote }} + {{- if .Values.metricsListener.enabled }} + - name: metrics + containerPort: {{ .Values.metricsListener.port }} + protocol: TCP + {{- end }} {{- with .Values.health.readiness }} readinessProbe: {{- /* /ready fails while any backend cluster is down. */}} @@ -286,6 +302,11 @@ spec: mountPath: {{ .Values.gatewayConfig.listenerTls.mountPath | quote }} readOnly: true {{- end }} + {{- if .Values.metricsListener.enabled }} + - name: metrics-tls + mountPath: {{ .Values.metricsListener.mountPath | quote }} + readOnly: true + {{- end }} {{- if $withCA }} - name: validate-ca mountPath: {{ $validateCA.mountPath | quote }} @@ -407,6 +428,11 @@ spec: secret: secretName: {{ .Values.gatewayConfig.listenerTls.existingSecret | quote }} {{- end }} + {{- if .Values.metricsListener.enabled }} + - name: metrics-tls + secret: + secretName: {{ .Values.metricsListener.existingSecret | quote }} + {{- end }} {{- range .Values.credentials }} - name: credential-{{ .name }} secret: diff --git a/charts/praxis-gateway/templates/networkpolicy.yaml b/charts/praxis-gateway/templates/networkpolicy.yaml index 59e0b585a..1e2efb32b 100644 --- a/charts/praxis-gateway/templates/networkpolicy.yaml +++ b/charts/praxis-gateway/templates/networkpolicy.yaml @@ -33,4 +33,15 @@ spec: ports: - port: {{ .Values.port.containerPort }} protocol: {{ .Values.port.protocol }} + {{- if .Values.metricsListener.enabled }} + - from: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: {{ toJson .Values.metricsListener.fromNamespaces }} + ports: + - port: {{ .Values.metricsListener.port }} + protocol: TCP + {{- end }} {{- end }} diff --git a/charts/praxis-gateway/templates/service-metrics.yaml b/charts/praxis-gateway/templates/service-metrics.yaml new file mode 100644 index 000000000..f55384a72 --- /dev/null +++ b/charts/praxis-gateway/templates/service-metrics.yaml @@ -0,0 +1,24 @@ +{{- include "praxis-gateway.normalize" . }} +{{- if .Values.metricsListener.enabled }} +{{- /* ClusterIP of its own, so a LoadBalancer gateway Service never exposes metrics. */}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "praxis-gateway.fullname" . }}-metrics + labels: + {{- include "praxis-gateway.labels" . | nindent 4 }} + app.kubernetes.io/component: gateway-metrics + {{- with .Values.metricsListener.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + ports: + - name: metrics + port: {{ .Values.metricsListener.port }} + targetPort: metrics + protocol: TCP + selector: + {{- include "praxis-gateway.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/praxis-gateway/templates/servicemonitor.yaml b/charts/praxis-gateway/templates/servicemonitor.yaml new file mode 100644 index 000000000..8355592af --- /dev/null +++ b/charts/praxis-gateway/templates/servicemonitor.yaml @@ -0,0 +1,35 @@ +{{- include "praxis-gateway.normalize" . }} +{{- with .Values.metricsListener }} +{{- if and .enabled .serviceMonitor.enabled }} +{{- $name := printf "%s-metrics" (include "praxis-gateway.fullname" $) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ $name }} + labels: + {{- include "praxis-gateway.labels" $ | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "praxis-gateway.selectorLabels" $ | nindent 6 }} + app.kubernetes.io/component: gateway-metrics + endpoints: + - port: metrics + scheme: https + path: /metrics + interval: {{ .serviceMonitor.interval }} + tlsConfig: + serverName: {{ printf "%s.%s.svc" $name $.Release.Namespace }} + ca: + configMap: + name: {{ .serviceMonitor.caConfigMap.name | quote }} + key: {{ .serviceMonitor.caConfigMap.key | quote }} + metricRelabelings: + # Praxis names the upstream cluster "cluster", which ACM sets to the managed cluster. + - sourceLabels: [cluster] + regex: "(.+)" + targetLabel: backend + - action: labeldrop + regex: cluster +{{- end }} +{{- end }} diff --git a/charts/praxis-gateway/tests/metrics_listener_test.yaml b/charts/praxis-gateway/tests/metrics_listener_test.yaml new file mode 100644 index 000000000..0cbd48990 --- /dev/null +++ b/charts/praxis-gateway/tests/metrics_listener_test.yaml @@ -0,0 +1,159 @@ +suite: metrics listener +templates: + - templates/deployment.yaml + - templates/service-metrics.yaml + - templates/servicemonitor.yaml + - templates/networkpolicy.yaml + - templates/service.yaml +release: + name: gw + namespace: grid +set: + image.flavor: grid-gateway + image.repository: quay.io/sbatsche/grid-gateway + image.tag: t + networkPolicy.enabled: true + networkPolicy.from: + - podSelector: {} + metricsListener.enabled: true + metricsListener.existingSecret: grid-gateway-metrics-tls + metricsListener.fromNamespaces: [openshift-user-workload-monitoring, openshift-monitoring] +tests: + - it: renders nothing for metrics by default + set: + metricsListener.enabled: false + asserts: + - hasDocuments: + count: 0 + template: templates/service-metrics.yaml + - hasDocuments: + count: 0 + template: templates/servicemonitor.yaml + - notExists: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "GRID_METRICS_ADDR")] + template: templates/deployment.yaml + + - it: points the gateway at the listener address and the mounted cert + template: templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "GRID_METRICS_ADDR")].value + value: "0.0.0.0:9443" + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].env[?(@.name == "GRID_METRICS_TLS_KEY")].value + value: /etc/praxis/metrics-tls/tls.key + - equal: + path: spec.template.spec.containers[?(@.name == "praxis")].ports[?(@.name == "metrics")].containerPort + value: 9443 + - equal: + path: spec.template.spec.volumes[?(@.name == "metrics-tls")].secret.secretName + value: grid-gateway-metrics-tls + + - it: serves metrics on a ClusterIP Service of its own, never the gateway Service + set: + service.type: LoadBalancer + asserts: + - equal: + path: spec.type + value: ClusterIP + template: templates/service-metrics.yaml + - equal: + path: metadata.labels["app.kubernetes.io/component"] + value: gateway-metrics + template: templates/service-metrics.yaml + - notContains: + path: spec.ports + content: {name: metrics, port: 9443, targetPort: metrics, protocol: TCP} + template: templates/service.yaml + + - it: admits only the named namespaces to the metrics port + template: templates/networkpolicy.yaml + asserts: + - equal: + path: spec.ingress[1] + value: + from: + - namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [openshift-user-workload-monitoring, openshift-monitoring] + ports: + - port: 9443 + protocol: TCP + - lengthEqual: + path: spec.ingress + count: 2 + + - it: scrapes over verified TLS and renames the praxis cluster label to backend + template: templates/servicemonitor.yaml + set: + metricsListener.serviceMonitor.enabled: true + metricsListener.serviceMonitor.caConfigMap: {name: openshift-service-ca.crt, key: service-ca.crt} + asserts: + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: gateway-metrics + - equal: + path: spec.endpoints[0].tlsConfig.serverName + value: gw-praxis-gateway-metrics.grid.svc + - equal: + path: spec.endpoints[0].tlsConfig.ca.configMap.name + value: openshift-service-ca.crt + - equal: + path: spec.endpoints[0].metricRelabelings + value: + - sourceLabels: [cluster] + regex: "(.+)" + targetLabel: backend + - action: labeldrop + regex: cluster + + - it: refuses the listener without the NetworkPolicy + template: templates/deployment.yaml + set: + networkPolicy.enabled: false + asserts: + - failedTemplate: + errorPattern: "metricsListener needs networkPolicy.enabled" + + - it: refuses the listener without a cert + template: templates/deployment.yaml + set: + metricsListener.existingSecret: "" + asserts: + - failedTemplate: + errorPattern: "metricsListener.existingSecret is required" + + - it: refuses the listener without namespaces + template: templates/deployment.yaml + set: + metricsListener.fromNamespaces: [] + asserts: + - failedTemplate: + errorPattern: "metricsListener.fromNamespaces needs at least one namespace" + + - it: refuses the listener on the ai image + template: templates/deployment.yaml + set: + image.flavor: ai + image.repository: ghcr.io/praxis-proxy/ai + asserts: + - failedTemplate: + errorPattern: "metricsListener needs image.flavor grid-gateway" + + - it: refuses a metrics port the gateway already uses + template: templates/deployment.yaml + set: + metricsListener.port: 8080 + asserts: + - failedTemplate: + errorPattern: "metricsListener.port 8080 collides" + + - it: refuses a ServiceMonitor without a CA + template: templates/deployment.yaml + set: + metricsListener.serviceMonitor.enabled: true + asserts: + - failedTemplate: + errorPattern: "caConfigMap.name is required" diff --git a/charts/praxis-gateway/values.schema.json b/charts/praxis-gateway/values.schema.json index aa48826d3..dd7aaac04 100644 --- a/charts/praxis-gateway/values.schema.json +++ b/charts/praxis-gateway/values.schema.json @@ -188,6 +188,36 @@ "from": { "type": "array", "items": { "type": "object" } } } }, + "metricsListener": { + "type": "object", + "additionalProperties": false, + "description": "TLS listener serving only GET /metrics.", + "properties": { + "enabled": { "type": "boolean" }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "existingSecret": { "type": "string" }, + "mountPath": { "type": "string" }, + "fromNamespaces": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "service": { + "type": "object", + "additionalProperties": false, + "properties": { "annotations": { "type": "object" } } + }, + "serviceMonitor": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "interval": { "type": "string" }, + "caConfigMap": { + "type": "object", + "additionalProperties": false, + "properties": { "name": { "type": "string" }, "key": { "type": "string" } } + } + } + } + } + }, "replicaCount": { "type": "integer", "minimum": 1, diff --git a/charts/praxis-gateway/values.yaml b/charts/praxis-gateway/values.yaml index 57c56fc1e..46934ade4 100644 --- a/charts/praxis-gateway/values.yaml +++ b/charts/praxis-gateway/values.yaml @@ -292,6 +292,34 @@ networkPolicy: # - namespaceSelector: {matchLabels: {kubernetes.io/metadata.name: ingress}} # podSelector: {matchLabels: {app: authenticating-front}} +# -- TLS listener serving only GET /metrics, for an in-cluster Prometheus. The admin +# listener cannot be scraped from outside the pod. Needs the grid-gateway image and +# networkPolicy.enabled, since the port is reachable only from fromNamespaces. +metricsListener: + # -- Serve the listener. + enabled: false + # -- Container and Service port. + port: 9443 + # -- Secret holding tls.crt and tls.key, for example the serving cert that + # service.annotations requests on OpenShift. Required when enabled. + existingSecret: "" + # -- Mount path for the cert. The listener reloads it when it changes. + mountPath: "/etc/praxis/metrics-tls" + # -- Namespaces, by name, whose pods may reach the port. Required when enabled. + fromNamespaces: [] + service: + # -- Annotations on the ClusterIP metrics Service. + annotations: {} + serviceMonitor: + # -- Render a monitoring.coreos.com/v1 ServiceMonitor for the metrics Service. + enabled: false + # -- Scrape interval. + interval: 30s + # -- ConfigMap holding the CA that signed the metrics cert. Required when enabled. + caConfigMap: + name: "" + key: "ca.crt" + # -- Gateway Service configuration. service: # -- Create a Service for the gateway. diff --git a/deploy/crds/inferenceprovider.yaml b/deploy/crds/inferenceprovider.yaml index 910be4c90..f6e802a39 100644 --- a/deploy/crds/inferenceprovider.yaml +++ b/deploy/crds/inferenceprovider.yaml @@ -16,8 +16,19 @@ spec: - jsonPath: .spec.providerKind name: Provider type: string + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Reason + priority: 1 + type: string - jsonPath: .status.phase name: Phase + priority: 1 type: string name: v1alpha1 schema: @@ -281,6 +292,17 @@ spec: - message: set exactly one of caSecretRef and caConfigMapRef rule: has(self.caSecretRef) != has(self.caConfigMapRef) type: object + maxRunning: + description: |- + Most requests one endpoint runs at once: vLLM max-num-seqs. + + The site's operator multiplies it by the EPP's fresh ready endpoints and publishes + the product as `grid_provider_capacity_requests`, the capacity gateways weigh the + provider's load against. When absent, or no endpoint is fresh, capacity is unknown. + format: uint32 + minimum: 1.0 + nullable: true + type: integer metricsConfig: description: |- Prometheus metrics scraping configuration. @@ -399,6 +421,14 @@ spec: description: Metric name for normalised queue depth (0.0–1.0). nullable: true type: string + readyEndpoints: + description: |- + Metric name counting the pool's ready endpoints, read for the `Ready` condition. + + Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`. + Filtered by `poolName` when set. Zero marks the provider not ready. + nullable: true + type: string type: object staleMetricsSeconds: description: |- @@ -756,6 +786,46 @@ spec: description: Observed status of an [`InferenceProvider`]. nullable: true properties: + conditions: + description: |- + Observed conditions. `Ready` says whether the provider can currently serve a request. + + Written by the operator's signals loop, never by provider reconciliation. + items: + description: One observed condition, shaped like `metav1.Condition`. + properties: + lastTransitionTime: + description: When `status` last changed, RFC 3339. + format: date-time + type: string + message: + default: "" + description: Human-readable detail. + type: string + observedGeneration: + description: The `metadata.generation` this was computed against. + format: int64 + nullable: true + type: integer + reason: + description: CamelCase reason for the status. + type: string + status: + description: '`True`, `False`, or `Unknown`.' + type: string + type: + description: Condition type, such as `Ready`. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map matchingSites: default: [] description: Sites matched by the site selector. @@ -803,6 +873,17 @@ spec: `HealthCheckTlsIdentityMismatch` nullable: true type: string + state: + description: |- + `Ready`, `NotReady`, or `Unknown`: the `Ready` condition's status, for display. + + Written with the condition, never by provider reconciliation. + enum: + - Ready + - NotReady + - Unknown + nullable: true + type: string type: object required: - spec diff --git a/docs/README.md b/docs/README.md index 1247e558b..dc1fb74e2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,8 @@ - [Routing Guide](routing.md): choose a routing behavior and configure policy, scoring, selection groups, affinity, and selection modes. +- [Tuning cross-site site selection](site-selection.md): how a grid gateway + chooses a site, what feeds it, and what to change. - [Routing Architecture and Overlay Contract](architecture/routing.md): overlay format, revision lifecycle, scope, provenance, security, and delivery. - [Provider Scoring](architecture/scoring.md): metrics input, normalization, diff --git a/docs/architecture/crds.md b/docs/architecture/crds.md index 807b1beda..160806f4c 100644 --- a/docs/architecture/crds.md +++ b/docs/architecture/crds.md @@ -563,6 +563,54 @@ credential projection can become available. **Phases**: Pending → Available → Degraded → Unavailable +**Readiness**: `status.conditions` carries a `Ready` condition, written by the +operator's signals loop from each scrape of the provider's metrics. Readiness is +a condition, not a phase: `phase` follows the provider's configuration only, and +says nothing about whether it serves now. + +| Status | Reason | When | +|---|---|---| +| `True` | `Ready` | The latest scrape succeeded with at least one ready endpoint. | +| `False` | `NoEndpointsReady` | Two consecutive scrapes counted zero ready endpoints, and the EPP recorded no engine answer in the last 30s. | +| `False` | `NoLivenessCheck` | The scrape answered without the pool's ready-endpoint series (`llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`, for `poolName`). The message names what was missing. | +| `False` | `ScrapeTimedOut` | No scrape succeeded within `staleMetricsSeconds`, and the latest timed out. | +| `False` | `ScrapeUnauthorized` | As above, and the latest was refused with 401 or 403. | +| `False` | `TLSHandshakeFailed` | As above, and the latest failed TLS, including the TLS material. | +| `False` | `ScrapeFailed` | As above, and the latest failed otherwise. The message names the class: `dns`, `connect`, `http`, `body_cap`, `parse`, or `config`. | +| `False` | `MetricsStale` | No scrape succeeded within `staleMetricsSeconds`, and none failed. | +| `False` | `ProviderUnavailable` | The provider is `Unavailable`. | +| `Unknown` | `AwaitingFirstScrape` | No scrape has succeeded yet, within the grace window. | +| `Unknown` | `MetricsNotConfigured` | No `metricsConfig`, so readiness cannot be read. | + +The operator logs each change of reason once: at WARN when the provider turns not +ready, at INFO when it returns to `Ready` or waits. Each scrape counts in +`grid_provider_scrape_total{grid_provider,result}`, where `result` is `success`, +`no_series`, or a failure class, and +`grid_provider_last_scrape_success_timestamp_seconds{grid_provider}` holds the time +of the last scrape with the ready-endpoint series. + +A provider whose `Ready` is `False` is excluded from routing: its site publishes +`grid_provider_ready 0`, and its serving config entry carries `admission: none`. +See [Polling Cross-Site Load Signals](polling-metrics.md#provider-readiness). + +`status.state` repeats the condition's status as one word for display, `Ready`, +`NotReady`, or `Unknown`, so `kubectl get inferenceproviders` reads like +`kubectl get nodes`. The STATUS column shows it. `-o wide` adds REASON, the +condition's reason, and PHASE. + +```text +NAME PROVIDER STATUS AGE +qwen3-site-a self_hosted Ready 3d +qwen3-site-b self_hosted NotReady 3d +``` + +`spec.maxRunning` is the most requests one endpoint runs at once, vLLM +max-num-seqs. Its site's operator multiplies it by the EPP's fresh ready endpoints +and publishes the product as `grid_provider_capacity_requests`, the capacity gateways weigh +the provider's load against. When it is absent, or no endpoint is fresh, capacity is +unknown. Before this release `maxRunning` counted the whole pool: reinstall with +per-endpoint values. + `spec.capacityWeight` is an optional positive relative provider capacity from `1` through `1000`, used only with `GridNetwork.spec.selectionPolicy.mode: weightedRandom` and `placementPolicy.strategy: static`. If omitted, the @@ -651,6 +699,7 @@ routing architecture for full semantics. | `prefixCacheHitRatio` | Prefix-cache hit ratio from `0.0` to `1.0`. | | `errorRate` | Error rate from `0.0` to `1.0`. | | `healthy` | Health gauge interpreted by the metrics parser. | +| `readyEndpoints` | Ready endpoints in the pool, read for the `Ready` condition. Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`, filtered by `poolName`. | #### TLS and mTLS @@ -739,8 +788,10 @@ Whoever runs the EPP receives the token, and it is valid against the API server, the operator never sends its own token. The grid-operator chart creates a scraper ServiceAccount allowed only `get` on the nonResourceURL `/metrics` (`rbac.metricsScraper`, default `true`), and the operator mints a 10-minute token for it -with the TokenRequest API, bound to the operator Pod, reusing it until two thirds of its -lifetime has passed (about 400 seconds). The token is never logged. +with the TokenRequest API, reusing it until two thirds of its lifetime has passed (about +400 seconds). The token is not bound to a Pod: the API server binds a token only to a Pod +running as the token's own ServiceAccount, and the operator runs as another. The token is +never logged. A failed mint logs at WARN when the error changes. A credential goes only to a host proven by the CA `metricsConfig.tls` names: with `auth` set, an `https://` endpoint without `tls` is refused rather than trusted through diff --git a/docs/architecture/polling-metrics.md b/docs/architecture/polling-metrics.md index c026acd91..d7d9859c0 100644 --- a/docs/architecture/polling-metrics.md +++ b/docs/architecture/polling-metrics.md @@ -21,6 +21,7 @@ one verified identity. | Verify | The peer's certificate is checked against the Grid CA and its SPIFFE identity, then the verified identity is compared to the site the poller intended to reach. A valid Grid peer answering for a site the poller did not dial is refused. | | Read | The exposition body is read under a byte ceiling and a time bound, so a slow or oversized peer cannot hold the poll open or exhaust memory. | | Store | Each reading is keyed on the verified peer identity, never a value the response body carries. A body label that disagrees with the verified owner is dropped. | +| Bound | An operator keeps from a peer only the signal contract names and the EPP pool averages the gateway routes on, a `grid_provider` that is a DNS-1123 label, a finite non-negative value (at most one for `grid_provider_ready` and `grid_provider_error_ratio`), and its first 64 providers in name order. A peer's custom `signalNames` are dropped. Each refusal counts in `grid_peer_signals_refused_total{peer,reason}`. | The verified peer identity is the store key. The response body cannot choose where its readings land, so one peer cannot inject readings attributed to @@ -50,6 +51,118 @@ the store and produces a candidate list ordered least-loaded-first. The request path reads one ordered snapshot and takes the front admitted candidate. The request path does not read raw signals or compute load. It reads resolved order. +## Provider Readiness + +Each site's operator decides whether each of its providers can serve now and +publishes the verdict as `grid_provider_ready{grid_site,grid_provider}`: 1 when +ready, 0 when not. A provider is not ready when its EPP reports zero ready +endpoints for two scrapes in a row and recorded no engine answer in the last 30s +(the EPP counts endpoints with fresh metrics, so a saturated engine can read as +zero while it serves), when a scrape answers without the pool's +ready-endpoint series, when no scrape has succeeded within `staleMetricsSeconds` +(half the signal TTL when unset), or when the provider is `Unavailable`. The same +verdict is the provider's `Ready` condition, whose reason names the cause and, for a +failed scrape, its class. See crds.md for the reasons and the scrape metrics. + +The gateway reads the latest readiness sample for each candidate when it orders +the snapshot. A candidate whose latest sample is 0 is excluded, so a site is +dropped within one poll of its operator deciding, and readmitted within one poll +of it recovering. A missing series reads as ready, so a site whose operator +predates readiness is still routed. The serving config carries the same verdict +for this site's own providers as `admission: none`. + +When every candidate for a model is excluded, the gateway answers 503 with +`Retry-After`, not 404: the model exists but cannot be served now. + +With the defaults of a 5 s scrape and a 5 s poll, exclusion takes at most about +15 s and rejoin about 10 s. + +## Provider In-flight + +Each site's operator also publishes how many requests each provider holds, as +`grid_provider_in_flight_requests{grid_site,grid_provider}`. Every input comes from the EPP's +`/metrics`, and nothing scrapes vLLM. The value is the larger of two estimates, plus the +requests the EPP's flow control holds for the pool (`llm_d_epp_flow_control_queue_size`): + +- The EPP's per-endpoint `llm_d_epp_inflight_requests`, summed, taking each endpoint's + largest count across producer instances. It needs the EPP's inflight-load-producer. +- The pool's average running plus average queued requests, times ready endpoints. + +The larger, so an EPP restart that zeroes its count does not make the site look idle. +The per-endpoint count carries no pool label, so when one EPP serves more than one pool the +operator uses the pool averages alone. A site with no fresh endpoint publishes nothing, +since its averages are frozen at their last value, and the gateway reads it as unknown. + +On a prefill/decode pool the EPP counts a request on its prefill and its decode +endpoint, so the value counts endpoint occupancy, up to twice the requests. Capacity +must therefore count the slots of every ready endpoint, prefill and decode alike. + +Point `metricsConfig.metricsEndpoint` at the EPP Service. A pod or headless address +can reach a standby replica, which reports no series. + +Capacity is the provider's per-endpoint `spec.maxRunning` times the EPP's fresh ready +endpoints, published as `grid_provider_capacity_requests`, so it shrinks when pods are lost. +With no fresh endpoint it is unpublished, and the gateway reads it as unknown. + +With both known, the operator also publishes their ratio, requests held over +capacity, as `grid_provider_saturation_ratio`. Gateways choose sites by it. + +## Provider Latency + +Each site's operator also publishes recent latency per provider, from the EPP's request +histograms over the last 30s. A value is published only when at least 20 requests +completed in that window, and it is never borrowed from another site: + +- `grid_provider_ttft_p50_seconds` and `grid_provider_ttft_p90_seconds`: time to first + token for streaming requests, from the EPP's `llm_d_epp_request_ttft_seconds`. The EPP + times it from receiving the request, so it includes flow-control wait and network. +- `grid_provider_tpot_seconds`: mean time per output token for streaming requests. +- `grid_provider_prefill_seconds_per_token`: mean TTFT less mean flow-control wait, over + mean uncached input tokens. It is an estimate that moves with the workload mix: TTFT + counts streaming requests only, while token counts cover every request. +- `grid_provider_error_ratio`: failed requests over all requests. The latency histograms + record only successful requests, so a failing site can read fast. This shows it. + +The EPP labels these histograms by model, not pool, so an EPP serving more than one pool +reports their combined latency. An EPP restart resets its counters, and the window starts +over from the next scrape. + +## Provider Series on /metrics + +The operator also exports the provider series on its Prometheus `/metrics` listener, +labeled `grid_site` and `grid_provider`: readiness, in-flight requests, capacity, +saturation, and the latency series above. It exports its own providers and those it +polls from peers, so a Prometheus that scrapes one hub sees every site. A peer's value +there is what the hub last polled, up to one peer poll old, not what the peer's own +operator holds now. A series the operator does not hold is absent, not 0. + +## Site Selection + +The gateway chooses a site by each provider's load, rho, the saturation its operator +publishes (`grid_provider_saturation_ratio`), and draws in proportion to its published +capacity (`grid_provider_capacity_requests`). It reads both when it orders the snapshot, +so a request only filters and draws. A site has room when rho is below 1. Among healthy +sites with room: + +- Three or more: two distinct sites are drawn in proportion to capacity, and the one + with the lower rho is chosen. The busiest of three is never chosen. +- Two: one draw, each site weighted by capacity over 1 + rho, with each share kept + between a tenth and nine tenths, so a recovering site is still measured. +- One: that site. + +With no site that has room, the gateway draws by capacity among the healthy sites tied +on the best polled queue depth. That also covers sites that publish no in-flight count +or capacity: a site without either has no load input, and sites without capacity are +drawn evenly. A cluster praxis reports with no healthy endpoint is never drawn while a +healthy one is left. + +A model sheds when every healthy site serving it has a fresh rho of at least 1.05. It +routes again once one reaches 0.95. A shed request gets 429 with `Retry-After` and an +OpenAI-style error, type `rate_limit_exceeded` and code `capacity_exhausted`, so a +client backs off the way it does for any overload. A model with no healthy routable +site gets 503 with code `no_healthy_site`: an outage, not load. The gateway keeps no +count of its own requests in flight. + ## Failure Behavior | Condition | Signal produced | Routing effect | @@ -58,6 +171,7 @@ request path does not read raw signals or compute load. It reads resolved order. | Readings all stale (older than the window) | Same as no reading. | The candidate sorts last until a fresh reading arrives. | | Peer unreachable or slow | The poll returns an error and no reading is written. | The candidate ages out of the window and then sorts last. The poll loop continues, and one unreachable peer does not wedge the others. | | Peer presents an untrusted or mismatched certificate | The connection is refused, so no reading is written. | The peer contributes nothing to the order. | +| Peer reports `grid_provider_ready 0` | The candidate is excluded. | It takes no new requests until a later reading says 1. If every candidate is excluded, the model answers 503. | Loss of signal degrades to "least preferred," never to "silently treated as idle." A drained burst stays penalized until it ages out of the window rather diff --git a/docs/routing.md b/docs/routing.md index 348833d49..f3bbd7cde 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -2,6 +2,8 @@ AI Grid Network (AGN) routes inference traffic across provider gateways using a **multi-dimensional policy** rather than a single load-balancing algorithm. This guide starts with the routing outcome you want, then shows how policy, scoring, groups, affinity, and selection mode work together. +This guide covers the routing overlay and the consumer Praxis `intelligent_route` filter. A grid gateway serving `gridServing` chooses sites with `grid_site_route` instead, from load each site publishes: see [Tuning cross-site site selection](site-selection.md). + The practical model is: 1. Determine which providers are eligible to receive the request. diff --git a/docs/site-selection.md b/docs/site-selection.md new file mode 100644 index 000000000..253947bcb --- /dev/null +++ b/docs/site-selection.md @@ -0,0 +1,132 @@ +# Tuning cross-site site selection + +This guide covers how a grid gateway serving `gridServing` chooses a site for each +request in `grid_site_route`, what feeds that choice, and what you can change. The +routing overlay and the consumer Praxis `intelligent_route` filter, with selection +groups and scoring policies, are a separate path described in +[routing.md](routing.md). + +## Site choice + +Each site's operator scrapes its EPP and publishes, per provider, how many requests +the provider holds and how many it can run at once. The gateway polls every site and +computes each site's load, rho, as requests held over capacity. A site has room when +rho is below 1. + +Among healthy sites with room: + +- **Three or more:** the gateway draws two sites in proportion to capacity and takes the + one with lower rho. +- **Two:** one draw, each site weighted by capacity over 1 + rho, each share kept between + 0.1 and 0.9. +- **One:** that site. + +With no site that has room, the gateway draws by capacity among the healthy sites tied +on the best polled queue depth. When every healthy site has rho of at least 1.05, the +model sheds: requests get 429 with `Retry-After` until a site reaches 0.95. + +A healthy site is ready and has a backend praxis reports healthy. A site that is not +ready, or whose cluster has no healthy endpoint, is chosen only when no healthy site is +left. + +## Inputs + +| Input | Published as | Source | +|---|---|---| +| Requests held | `grid_provider_in_flight_requests` | The larger of the EPP's `llm_d_epp_inflight_requests` summed over endpoints, and its average running plus queued requests times ready endpoints, plus requests its flow control holds (`llm_d_epp_flow_control_queue_size`) | +| Capacity | `grid_provider_capacity_requests` | `spec.maxRunning` times the EPP's fresh ready endpoints (`llm_d_epp_ready_endpoints`) | +| Load (rho) | `grid_provider_saturation_ratio` | Requests held over capacity, computed by the operator when both are known | +| Readiness | `grid_provider_ready` | 0 after two scrapes with no ready endpoints, or no successful scrape within `staleMetricsSeconds` | +| Queue depth | `llm_d_epp_average_queue_size` | The EPP, republished as is | + +Every input comes from the EPP's `/metrics`. Point `metricsConfig.metricsEndpoint` at +the EPP Service, since a pod address can reach a standby replica that reports nothing. +In-flight counts need the EPP's inflight-load-producer. Without it the pool averages +stand in, and they miss requests the EPP holds. + +A provider with no `maxRunning`, or no fresh endpoint, publishes no capacity. Its rho +is unknown, and it takes requests only through the queue-depth overflow. + +## Latency inputs + +Each operator also publishes recent latency per provider, over the last 30 seconds, +when at least 20 requests completed: `grid_provider_ttft_p50_seconds`, +`grid_provider_ttft_p90_seconds`, `grid_provider_tpot_seconds`, +`grid_provider_prefill_seconds_per_token`, and `grid_provider_error_ratio`. The gateway +reads them, but site choice does not use them: it ranks by rho alone. They show why one +site is slower than another, and the error ratio shows a failing site, which the latency +series would read as fast. + +The EPP times TTFT from receiving the request, so it includes flow-control wait and +network. Prefill seconds per token is an estimate that moves with the workload mix. + +## Choosing maxRunning + +Set `spec.maxRunning` to the most requests one endpoint runs at once, which is the +engine's `--max-num-seqs`. The operator multiplies it by ready endpoints, so it follows +scale-up and pod loss. + +An engine can run out of KV cache before it reaches max-num-seqs. vLLM logs its KV +concurrency at startup, as maximum concurrency for its configured tokens per request. +If your typical request is long, set `maxRunning` to the smaller of max-num-seqs and the +concurrency the KV cache holds at your typical request length. Too high a value makes a +KV-bound site look roomier than it is. + +On a prefill/decode pool, the EPP counts a request on both its prefill and its decode +endpoint, and ready endpoints count both roles. Use the same per-endpoint value for +both roles. + +## Settings + +| Setting | Default | Where | Effect of changing it | +|---|---|---|---| +| `spec.maxRunning` | none | InferenceProvider | Sets capacity per endpoint. Without it, rho is unknown. | +| `metricsConfig.metricsEndpoint` | none | InferenceProvider | The EPP Service to scrape. Required for every input. | +| `metricsConfig.staleMetricsSeconds` | half the signal TTL | InferenceProvider | How long a failing scrape keeps the last readiness before the provider reads not ready. | +| `GRID_SIGNALS_SCRAPE_INTERVAL_SECS` | 5 | Operator environment | How often the operator scrapes its EPP. Faster reacts sooner and loads the EPP more. | +| `metricsListener.enabled` | false | praxis-gateway chart | Serves the decision counters on a TLS listener, with a Service and ServiceMonitor. | + +The rest is fixed: + +- The gateway polls each site every 5s with a 2s timeout. A load sample is fresh for + two polls, 10s. +- Shedding starts when every healthy site holds rho of at least 1.05 and stops when one + reaches 0.95. +- A two-site draw keeps each share between 0.1 and 0.9. +- One request weighs at most 64 sites with room. +- A shed 429 and an outage 503 each carry a Retry-After of 3 to 7 seconds. + +## What to watch + +- `grid_route_decisions_total{site,reason}` on the gateway's metrics listener. `routed` + and `fallback` count by site. Refusals count under an empty site as `not_ready`, + `no_route`, `bad_request`, or `shed`. +- `grid_route_site_score{site,cluster}`: the queue depth each candidate was ordered by. + NaN marks an excluded or demoted candidate. +- `grid_provider_in_flight_requests`, `grid_provider_capacity_requests`, + `grid_provider_saturation_ratio`, and `grid_provider_ready` on each site's signals + endpoint, `/v1/site/signals`, and on each operator's `/metrics`. A hub's `/metrics` + also carries what it polled from its peers. +- `x-grid-site` and `x-grid-backend` on every routed response. +- Operator WARN `serving config: refusing candidates whose site is not a DNS-1123 label`. + +## Symptoms + +| Symptom | Likely cause | Action | +|---|---|---| +| Traffic piles onto one site | No capacity published, so the queue-depth overflow herds onto the site tied on the best queue for one poll window | Set `maxRunning` on every provider and check `grid_provider_capacity_requests` appears for each | +| A site is never chosen | Not ready, or its cluster has no healthy endpoint. Its `grid_route_site_score` is NaN. | Check `grid_provider_ready` and the gateway's cluster health log line. | +| A small or slow site takes little | Expected: draws follow capacity, and two choices never pick the busiest of three | Raise its `maxRunning` only if the engine runs more | +| 429 `capacity_exhausted` under light load | rho reads high: `maxRunning` too low, or in-flight counts both P/D roles against a decode-only capacity | Compare `grid_provider_in_flight_requests` with what the engines run, and correct `maxRunning` | +| 503 `no_healthy_site` | Every site serving the model is not ready or has no route from this gateway | Check `grid_provider_ready` and the serving config candidates | +| Traffic moves between sites every few seconds | Load near capacity, read at 5s polls | Expected near full. Check capacity covers the load. | +| A peer never routes | Its site id is not a DNS-1123 label, and the operator refuses it | Look for the refused-site WARN and rename the site | + +## What you cannot tune + +The gateway keeps no count of its own requests. Load comes from what each site's +operator publishes, so it is as fresh as the scrape and poll, about 5 to 15 seconds. The +selection is coarse by design. It keeps each site near its capacity share and away from +full, and the site's EPP picks the endpoint within the site. No per-site weight or +threshold is settable. + diff --git a/gateway/Cargo.lock b/gateway/Cargo.lock index 9bf30a3b4..5cade8a9f 100644 --- a/gateway/Cargo.lock +++ b/gateway/Cargo.lock @@ -36,6 +36,7 @@ dependencies = [ "grid-signals-client", "http", "metrics", + "praxis-proxy-core", "praxis-proxy-filter", "praxis-proxy-tls", "rustls", @@ -43,6 +44,7 @@ dependencies = [ "tokio", "tokio-rustls", "tracing", + "tracing-subscriber", "yaml_serde", "zeroize", ] @@ -1381,12 +1383,18 @@ version = "0.1.4" dependencies = [ "ai-grid-filters", "certs", + "httparse", "praxis-ai-filters", "praxis-proxy", "praxis-proxy-core", "praxis-proxy-filter", + "praxis-proxy-protocol", + "praxis-proxy-tls", "rustls", + "tokio", + "tokio-rustls", "tracing", + "yaml_serde", ] [[package]] diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index e36c2b44a..2f8efaf77 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -314,11 +314,18 @@ praxis-proxy-core = "0.7.3" praxis-ai-filters = { git = "https://github.com/praxis-proxy/ai.git", rev = "17b41771fcd53b22ab5c6f9badede0de9021404c", default-features = false } ai-grid-filters = { path = "ai-grid-filters" } tracing = "0.1.44" +praxis-protocol = { version = "0.7.3", package = "praxis-proxy-protocol" } +praxis-tls = { version = "0.7.3", package = "praxis-proxy-tls" } +tokio = { version = "1", features = ["io-util", "net", "rt", "sync", "time"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } +httparse = "1.10" [dev-dependencies] # The reload test runs a real mTLS signals peer against the binary. certs = { path = "../certs", default-features = false, features = ["verifier"] } rustls = { version = "0.23.45", features = ["ring"] } +serde_yaml = { version = "0.10", package = "yaml_serde" } +tokio = { version = "1", features = ["macros"] } # machete can't see these: used as `praxis` / `praxis_core` / `praxis_filter`, not their package names. [package.metadata.cargo-machete] diff --git a/gateway/ai-grid-filters/Cargo.toml b/gateway/ai-grid-filters/Cargo.toml index bade9a450..7586948d3 100644 --- a/gateway/ai-grid-filters/Cargo.toml +++ b/gateway/ai-grid-filters/Cargo.toml @@ -8,7 +8,9 @@ license.workspace = true publish = false [dependencies] -praxis-filter = { version = "0.7.0", package = "praxis-proxy-filter" } +praxis-filter = { version = "0.7.1", package = "praxis-proxy-filter" } +praxis-core = { version = "0.7.1", package = "praxis-proxy-core" } +praxis-tls = { version = "0.7.1", package = "praxis-proxy-tls" } grid-signals = { path = "../../signals" } # The gateway wiring spawns the pollers and builds their mTLS scrapers, so the # poller and cert issuance are real dependencies, not just test ones. @@ -25,6 +27,7 @@ http = "1" rustls = { version = "0.23", default-features = false, features = ["logging", "std", "tls12"] } metrics = "0.24.6" zeroize = "1.9.0" +tokio = { version = "1", features = ["net", "rt", "time"] } [dev-dependencies] # The flow tests bind a mock TLS peer to drive the real scrape path. @@ -32,6 +35,7 @@ tokio = { version = "1", features = ["macros", "net", "rt", "rt-multi-thread", " tokio-rustls = "0.26" rustls = { version = "0.23", features = ["ring"] } praxis-tls = { version = "0.7.1", package = "praxis-proxy-tls" } +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "std"] } [lints] workspace = true diff --git a/gateway/ai-grid-filters/src/control.rs b/gateway/ai-grid-filters/src/control.rs index ce94978d7..1c913dcdd 100644 --- a/gateway/ai-grid-filters/src/control.rs +++ b/gateway/ai-grid-filters/src/control.rs @@ -18,8 +18,9 @@ use praxis_filter::FilterError; use crate::{ descriptor::{RouteCandidate, validate_candidates, validate_local_site}, + health::ClusterHealth, serving::{GridServingConfig, PeerServingConfig, validate_peer}, - snapshot::RouteSnapshot, + snapshot::{Gauged, RouteSnapshot}, }; /// The order step a poller runs after each scrape. @@ -45,22 +46,43 @@ impl Topology { /// Validate the topology half of `config`. fn from_config(config: &GridServingConfig) -> Result { validate_local_site(&config.local_site)?; + let base = validate_candidates(config.candidates.clone())?; Ok(Self { - base: Arc::from(validate_candidates(config.candidates.clone())?), + base: Arc::from(base), local_site: Arc::from(config.local_site.as_str()), load_window_ms: config.load_window_ms, }) } - /// The snapshot ordered from `store` at `now`. - fn order(&self, store: &LoadStore, now: i64) -> RouteSnapshot { - RouteSnapshot::from_store( + /// The snapshot ordered from `store` at `now`, down clusters and peer gateways last, with the + /// models to shed given the set `shedding` the previous snapshot shed. + #[expect( + clippy::too_many_arguments, + reason = "each input is a distinct piece of control state" + )] + fn order( + &self, + store: &LoadStore, + now: i64, + health: &ClusterHealth, + shedding: &std::collections::BTreeSet>, + gauged: &mut Gauged, + ) -> RouteSnapshot { + let ordered = RouteSnapshot::from_store( self.base.iter().cloned().collect(), Arc::clone(&self.local_site), store, now, self.load_window_ms, - ) + ); + // Praxis demotes a cluster only once it has reported health. Without a registry the + // gateway knows nothing about backends, so it demotes nothing rather than guessing. + let ordered = if health.observed() { + ordered.demote(&health.down()) + } else { + ordered + }; + ordered.shed(shedding).published(gauged) } } @@ -110,8 +132,9 @@ pub(crate) struct Control { /// The topology every refresh orders, swapped on reload. topology: Arc>, - /// Serializes snapshot stores between the refreshes and a reload. - swap: Arc>, + /// Serializes snapshot stores between the refreshes and a reload, and holds the + /// site/cluster pairs with a published score. + swap: Arc>, /// Store retention, fixed for the process. window_secs: u64, @@ -127,6 +150,9 @@ pub(crate) struct Control { /// Builds and spawns a peer's poller. start: StartPeer, + + /// Backend clusters with no healthy endpoint, ordered last on every refresh. + health: Arc, } impl Control { @@ -134,23 +160,36 @@ impl Control { pub(crate) fn new(config: &GridServingConfig, start: StartPeer) -> Result { let topology = Topology::from_config(config)?; // Cold start: config order until the first poll re-orders it by live load. + let mut gauged = Gauged::new(); let cold_start = RouteSnapshot::from_static( topology.base.iter().cloned().collect(), Arc::clone(&topology.local_site), - ); + ) + .published(&mut gauged); Ok(Self { store: Arc::new(LoadStore::new(Duration::from_secs(config.window_secs))), snapshot: Arc::new(ArcSwap::from_pointee(cold_start)), topology: Arc::new(ArcSwap::from_pointee(topology)), - swap: Arc::new(Mutex::new(())), + swap: Arc::new(Mutex::new(gauged)), window_secs: config.window_secs, peers: HashMap::new(), applied: None, identity: None, start, + health: Arc::default(), }) } + /// The cluster health the route filter publishes into and every refresh reads. + pub(crate) fn health(&self) -> Arc { + Arc::clone(&self.health) + } + + /// The live load store. + pub(crate) fn store(&self) -> Arc { + Arc::clone(&self.store) + } + /// The snapshot the filter reads. pub(crate) fn snapshot(&self) -> Arc> { Arc::clone(&self.snapshot) @@ -176,7 +215,9 @@ impl Control { return Ok(None); } // Validated before any poller starts, so an invalid config starts and drops nothing. - let topology = Arc::new(validate_config(config)?); + validate_config(config)?; + // Built after validation: it reads the identity from disk, so a failure here is retried, never settled. + let topology = Arc::new(Topology::from_config(config)?); if config.window_secs != self.window_secs { tracing::warn!( current = self.window_secs, @@ -186,11 +227,14 @@ impl Control { } let (next, started) = self.start_changed(config, renewed)?; let outcome = self.reconcile(&next, started, renewed, config.load_window_ms); - let ordered = Arc::new(topology.order(&self.store, now_ms())); { - let _swapping = self.swap.lock().unwrap_or_else(PoisonError::into_inner); + // Order under the lock, so a concurrent refresh never publishes an older down set or load. + let mut gauged = self.swap.lock().unwrap_or_else(PoisonError::into_inner); + let shedding = self.snapshot.load().shedding.clone(); + let ordered = Arc::new(topology.order(&self.store, now_ms(), &self.health, &shedding, &mut gauged)); self.topology.store(topology); self.snapshot.store(ordered); + drop(gauged); } // The reload stands and its topology is published: only now may the new pollers write, // so their first refresh orders the new topology. Committing a kept poller is a no-op. @@ -271,19 +315,21 @@ impl Control { } /// The refresh a poller runs each cycle: order the current topology by load. - fn refresh(&self) -> Refresh { + pub(crate) fn refresh(&self) -> Refresh { make_refresh( Arc::clone(&self.topology), Arc::clone(&self.snapshot), Arc::clone(&self.swap), + Arc::clone(&self.health), now_ms, ) } } /// The topology of `config`, refusing what no retry can fix: a bad candidate, peer, or duplicate site. -fn validate_config(config: &GridServingConfig) -> Result { - let topology = Topology::from_config(config)?; +fn validate_config(config: &GridServingConfig) -> Result<(), FilterError> { + validate_local_site(&config.local_site)?; + validate_candidates(config.candidates.clone())?; let mut sites = std::collections::HashSet::with_capacity(config.peers.len()); for peer in &config.peers { validate_peer(peer)?; @@ -291,52 +337,37 @@ fn validate_config(config: &GridServingConfig) -> Result return Err(format!("grid: peer site {} appears twice", peer.site).into()); } } - Ok(topology) + Ok(()) } -/// Order the current topology and swap it in, unless a reload replaced it meanwhile. +/// Order the current topology and swap it in. +/// +/// Every refresher orders under `swap`, so the last store always reflects the latest +/// topology, down set, and load. pub(crate) fn make_refresh( topology: Arc>, snapshot: Arc>, - swap: Arc>, + swap: Arc>, + health: Arc, now: N, ) -> Refresh where N: Fn() -> i64 + Send + 'static, { Box::new(move |store: &LoadStore| { - let current = topology.load_full(); - let ordered = Arc::new(current.order(store, now())); - let _swapping = swap.lock().unwrap_or_else(PoisonError::into_inner); - if Arc::ptr_eq(¤t, &topology.load()) { - snapshot.store(ordered); - } + let mut gauged = swap.lock().unwrap_or_else(PoisonError::into_inner); + let shedding = snapshot.load().shedding.clone(); + let ordered = topology.load().order(store, now(), &health, &shedding, &mut gauged); + snapshot.store(Arc::new(ordered)); + drop(gauged); }) } /// A change detector over every identity file `config` names: the grid CA, /// client certificate, and key. Not a security function. /// -/// A rotation swaps the files at once but they are read one by one, so a pass -/// that straddles the swap mixes versions. Read until two passes agree. +/// Each file is hashed on its own, so no buffer holds the concatenated key. fn identity_digest(config: &GridServingConfig) -> [u8; 32] { - let mut last = read_identity(config); - for _ in 0..IDENTITY_READS { - let next = read_identity(config); - if next == last { - break; - } - last = next; - } - last -} - -/// Passes [`identity_digest`] makes after the first; a rotation tears at most one. -const IDENTITY_READS: usize = 3; - -/// One pass over the identity files. Each file is hashed on its own, so no -/// buffer holds the concatenated key. -fn read_identity(config: &GridServingConfig) -> [u8; 32] { let paths: std::collections::BTreeSet<&str> = config .peers .iter() @@ -448,7 +479,7 @@ pub(crate) fn watch(control: Arc>, path: PathBuf, every: Duration fn apply_file(control: &Mutex, path: &std::path::Path, bytes: &[u8], tally: &WatchCounts) -> bool { let parsed = serde_yaml::from_slice::(bytes) .map_err(|error| -> FilterError { format!("grid: parsing {}: {error}", path.display()).into() }) - .and_then(|config| validate_config(&config).map(|_| config)); + .and_then(|config| validate_config(&config).map(|()| config)); let (result, settled) = match parsed { Ok(config) => { let result = control.lock().unwrap_or_else(PoisonError::into_inner).apply(&config); @@ -493,7 +524,8 @@ fn report(result: &Result, FilterError>, tally: &WatchCoun kept = outcome.kept, "{applied}" ), - Ok(None) => tracing::info!("grid: serving config unchanged; peer pollers reused"), + // Nothing changed, so nothing to say at info: the counter still records the attempt. + Ok(None) => tracing::debug!("grid: serving config unchanged; peer pollers reused"), Err(error) => tracing::warn!(%error, "grid: serving config rejected; keeping the last good config"), } } @@ -508,7 +540,10 @@ fn report(result: &Result, FilterError>, tally: &WatchCoun reason = "tests; the pollers run on their own threads, so these sync tests wait with thread::sleep" )] mod tests { - use std::{sync::atomic::AtomicI64, time::Instant}; + use std::{ + sync::atomic::{AtomicBool, AtomicI64}, + time::Instant, + }; use grid_signals_client::{FetchError, Scrape, SignalSource, spawn_on_thread_held}; @@ -597,6 +632,7 @@ mod tests { fn candidate(site: &str) -> CandidateConfig { CandidateConfig { + admission: crate::descriptor::AdmissionState::default(), cluster: format!("pool-{site}"), credential: None, fresh: true, @@ -620,6 +656,7 @@ mod tests { client_cert_path: "/etc/grid/tls.crt".to_owned(), client_key_path: "/etc/grid/tls.key".to_owned(), pins: Vec::new(), + gateway: None, } } @@ -638,58 +675,6 @@ mod tests { snapshot.candidates.iter().map(|c| c.site.to_string()).collect() } - /// The routable sites, sorted: which sites route, not their load order. - fn members(snapshot: &RouteSnapshot) -> Vec { - let mut sites = sites(snapshot); - sites.sort(); - sites - } - - /// A `Secret` or `ConfigMap` volume: each file resolves through `..data`, which - /// `write` swaps in one rename, as the kubelet does. - struct Mount { - dir: PathBuf, - generation: usize, - } - - impl Mount { - fn new(name: &str, files: &[(&str, &str)]) -> Self { - let dir = std::env::temp_dir().join(format!("{name}-{}", std::process::id())); - let _stale = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).expect("dir"); - let mut mount = Self { dir, generation: 0 }; - mount.write(files); - mount - } - - fn path(&self, name: &str) -> String { - self.dir.join(name).to_string_lossy().into_owned() - } - - /// Publish `files` together; earlier files are dropped, as in a new Secret version. - fn write(&mut self, files: &[(&str, &str)]) { - self.generation = self.generation.saturating_add(1); - let version = format!("..{}", self.generation); - std::fs::create_dir(self.dir.join(&version)).expect("version dir"); - for (name, content) in files { - std::fs::write(self.dir.join(&version).join(name), content).expect("write"); - let link = self.dir.join(name); - if std::fs::symlink_metadata(&link).is_err() { - std::os::unix::fs::symlink(format!("..data/{name}"), &link).expect("file link"); - } - } - let staged = self.dir.join("..data_tmp"); - std::os::unix::fs::symlink(&version, &staged).expect("data link"); - std::fs::rename(&staged, self.dir.join("..data")).expect("swap"); - } - } - - impl Drop for Mount { - fn drop(&mut self) { - let _removed = std::fs::remove_dir_all(&self.dir); - } - } - fn front(snapshot: &ArcSwap) -> Option { snapshot.load().candidates.first().map(|c| c.site.to_string()) } @@ -810,10 +795,8 @@ mod tests { #[test] fn an_invalid_config_keeps_the_last_good_one() { let peers = Peers::default(); - let mut control = Control::new(&config(&["east"]), peers.starter()).expect("control"); - control.apply(&config(&["east"])).expect("first apply"); - // The topology, not the snapshot: a poller's scrape re-publishes the snapshot. - let before = control.topology.load_full(); + let grid = runtime(&peers, &config(&["east"])); + let before = grid.snapshot().load_full(); let mut bad_candidate = config(&["east", "west"]); bad_candidate.candidates[1].name = String::new(); @@ -822,17 +805,14 @@ mod tests { let mut zero = config(&["east", "west"]); zero.peers[1].interval_ms = 0; for bad in [bad_candidate, twice, zero] { - control.apply(&bad).expect_err("an invalid config is rejected"); + grid.reload(&bad).expect_err("an invalid config is rejected"); } - assert!( - Arc::ptr_eq(&before, &control.topology.load_full()), - "a rejected config publishes no topology" - ); + // Content, not identity: a poll may republish the same order at any time. assert_eq!( - sites(&control.snapshot().load()), - ["east"], - "the routable sites are unchanged" + sites(&before), + sites(&grid.snapshot().load_full()), + "the snapshot keeps the last good topology" ); assert_eq!(peers.starts("west"), 0, "no poller started for a rejected config"); assert_eq!(peers.starts("east"), 1); @@ -1037,36 +1017,43 @@ mod tests { grid.reload(&config(&["east"])).expect("reload"); - assert_eq!(members(&in_flight), ["east", "west"], "the request keeps its snapshot"); + // A poller refresh may reorder the snapshot, so compare the set. + let mut kept = sites(&in_flight); + kept.sort(); + assert_eq!(kept, ["east", "west"], "the request keeps its snapshot"); assert_eq!(sites(&snapshot.load()), ["east"], "the next request sees the new one"); } #[test] - fn a_refresh_that_raced_a_reload_is_dropped() { - let first = Arc::new(ArcSwap::from_pointee( + fn a_refresh_orders_while_holding_the_swap_lock() { + // Ordering under the lock means a reload or health tick cannot publish between + // this cycle reading its inputs and storing its order. + let topology = Arc::new(ArcSwap::from_pointee( Topology::from_config(&config(&["east"])).expect("topology"), )); let snapshot = Arc::new(ArcSwap::from_pointee(RouteSnapshot::from_static( Vec::new(), Arc::from("local"), ))); - let reloaded = Arc::new(Topology::from_config(&config(&["west"])).expect("topology")); + let swap = Arc::new(Mutex::new(Gauged::new())); + let held = Arc::new(AtomicBool::new(false)); let refresh = { - let topology = Arc::clone(&first); + let (swap, held) = (Arc::clone(&swap), Arc::clone(&held)); make_refresh( - Arc::clone(&first), + topology, Arc::clone(&snapshot), - Arc::new(Mutex::new(())), + Arc::clone(&swap), + Arc::default(), move || { - // The reload lands while this cycle is ordering. - topology.store(Arc::clone(&reloaded)); + held.store(swap.try_lock().is_err(), Ordering::SeqCst); 1_000 }, ) }; refresh(&LoadStore::new(Duration::from_secs(60))); - assert!(sites(&snapshot.load()).is_empty(), "the stale order is not stored"); + assert!(held.load(Ordering::SeqCst), "ordered outside the swap lock"); + assert_eq!(sites(&snapshot.load()), ["east"]); } #[test] @@ -1081,9 +1068,13 @@ mod tests { let clock = Arc::new(AtomicI64::new(1_000)); let refresh = { let clock = Arc::clone(&clock); - make_refresh(topology, Arc::clone(&snapshot), Arc::new(Mutex::new(())), move || { - clock.load(Ordering::SeqCst) - }) + make_refresh( + topology, + Arc::clone(&snapshot), + Arc::new(Mutex::new(Gauged::new())), + Arc::default(), + move || clock.load(Ordering::SeqCst), + ) }; let line = |site: &str, value: f64, at: i64| { format!(r#"{LOAD_METRIC}{{grid_site="{site}",grid_provider="pool-{site}"}} {value} {at}"#) @@ -1136,7 +1127,7 @@ mod tests { write(&yaml(&["east", "west"])); eventually("the rewrite handled", || counts().applied() == 1); - assert_eq!(members(&snapshot.load()), ["east", "west"], "the rewrite applied"); + assert_eq!(sites(&snapshot.load()), ["east", "west"], "the rewrite applied"); eventually("west polled", || peers.fetches("west") > 0); drop(grid); @@ -1145,17 +1136,18 @@ mod tests { #[test] fn a_renewed_identity_restarts_the_pollers_with_no_config_change() { - let mut identity = Mount::new( - "grid-identity", - &[("ca.pem", "old"), ("tls.crt", "old"), ("tls.key", "old")], - ); + let dir = std::env::temp_dir().join(format!("grid-identity-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let file = |name: &str| dir.join(name).to_string_lossy().into_owned(); + for name in ["ca.pem", "tls.crt", "tls.key"] { + std::fs::write(file(name), "old").expect("write"); + } let mut serving = config(&["east"]); - serving.peers[0].grid_ca_path = identity.path("ca.pem"); - serving.peers[0].client_cert_path = identity.path("tls.crt"); - serving.peers[0].client_key_path = identity.path("tls.key"); - let yaml = serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml"); - let file = Mount::new("grid-identity-serving", &[("serving.yaml", &yaml)]); - let path = file.path("serving.yaml"); + serving.peers[0].grid_ca_path = file("ca.pem"); + serving.peers[0].client_cert_path = file("tls.crt"); + serving.peers[0].client_key_path = file("tls.key"); + let path = dir.join("serving.yaml"); + std::fs::write(&path, serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml")).expect("write"); let peers = Peers::default(); let mut grid = runtime(&peers, &serving); @@ -1168,59 +1160,23 @@ mod tests { let counts = || grid.watcher().expect("watching").counts(); eventually("the startup file seen", || counts().reused() == 1); - identity.write(&[("ca.pem", "old"), ("tls.crt", "renewed"), ("tls.key", "renewed")]); + for name in ["tls.crt", "tls.key"] { + std::fs::write(file(name), "renewed").expect("renew"); + } eventually("the renewal applied", || counts().applied() == 1); assert_eq!(peers.starts("east"), 2, "the poller restarted on the renewed identity"); std::thread::sleep(Duration::from_millis(100)); assert_eq!(counts().applied(), 1, "a settled identity restarts nothing more"); drop(grid); - } - - #[test] - fn an_identity_read_during_a_rotation_sees_one_version() { - let old = [("ca.pem", "ca"), ("tls.crt", "old"), ("tls.key", "old")]; - let new = [("ca.pem", "ca"), ("tls.crt", "new"), ("tls.key", "new")]; - let mut identity = Mount::new("grid-torn", &old); - let mut serving = config(&["east"]); - serving.peers[0].grid_ca_path = identity.path("ca.pem"); - serving.peers[0].client_cert_path = identity.path("tls.crt"); - serving.peers[0].client_key_path = identity.path("tls.key"); - let old_digest = identity_digest(&serving); - identity.write(&new); - let new_digest = identity_digest(&serving); - - // Each swap waits for two reads after the last, so no read spans two swaps, - // however long the reader stalls. A real rotation is hours apart. - let reads = Arc::new(AtomicUsize::new(0)); - let counted = Arc::clone(&reads); - let rotating = std::thread::spawn(move || { - for round in 0..40 { - let after = counted.load(Ordering::SeqCst).saturating_add(2); - let deadline = Instant::now().checked_add(Duration::from_secs(5)).expect("deadline"); - while counted.load(Ordering::SeqCst) < after && Instant::now() < deadline { - std::thread::yield_now(); - } - identity.write(if round % 2 == 0 { &old } else { &new }); - } - identity - }); - while !rotating.is_finished() { - let digest = identity_digest(&serving); - assert!( - digest == old_digest || digest == new_digest, - "read {} mixed two versions of the identity", - reads.load(Ordering::SeqCst) - ); - reads.fetch_add(1, Ordering::SeqCst); - } - drop(rotating.join().expect("rotation")); + std::fs::remove_dir_all(&dir).expect("cleanup"); } #[test] fn a_change_that_fails_on_unreadable_identity_is_retried_until_it_applies() { - let mut identity = Mount::new("grid-retry", &[]); - let cert = PathBuf::from(identity.path("tls.crt")); + let dir = std::env::temp_dir().join(format!("grid-retry-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let cert = dir.join("tls.crt"); let peers = Peers::default(); let ok = peers.starter(); let watched = cert.clone(); @@ -1233,39 +1189,39 @@ mod tests { let mut grid = crate::serving::start_runtime(&config(&["west"]), start).expect("runtime starts"); let mut serving = config(&["west", "east"]); serving.peers[1].client_cert_path = cert.to_string_lossy().into_owned(); - let yaml = serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml"); - let file = Mount::new("grid-retry-serving", &[("serving.yaml", &yaml)]); - let path = file.path("serving.yaml"); + let path = dir.join("serving.yaml"); + std::fs::write(&path, serde_yaml::to_string(&serde_yaml_value(&serving)).expect("yaml")).expect("write"); grid.watch(&path, Duration::from_millis(20)).expect("watch"); let counts = || grid.watcher().expect("watching").counts(); eventually("the change retried", || counts().rejected() >= 2); assert_eq!(sites(&grid.snapshot().load()), ["west"], "the old config stays"); - identity.write(&[("tls.crt", "issued")]); + std::fs::write(&cert, "issued").expect("issue"); eventually("the pending change applied", || counts().applied() == 1); - assert_eq!( - members(&grid.snapshot().load()), - ["east", "west"], - "the pending change applied" - ); + assert_eq!(sites(&grid.snapshot().load()), ["west", "east"]); drop(grid); + std::fs::remove_dir_all(&dir).expect("cleanup"); } #[test] fn a_refused_file_neither_blocks_renewal_nor_repeats_its_rejection() { - let mut identity = Mount::new( - "grid-refused", - &[("ca.pem", "old"), ("tls.crt", "old"), ("tls.key", "old")], - ); + let dir = std::env::temp_dir().join(format!("grid-refused-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("dir"); + let file = |name: &str| dir.join(name).to_string_lossy().into_owned(); + for name in ["ca.pem", "tls.crt", "tls.key"] { + std::fs::write(file(name), "old").expect("write"); + } let mut serving = config(&["east"]); - serving.peers[0].grid_ca_path = identity.path("ca.pem"); - serving.peers[0].client_cert_path = identity.path("tls.crt"); - serving.peers[0].client_key_path = identity.path("tls.key"); - let yaml = |config: &GridServingConfig| serde_yaml::to_string(&serde_yaml_value(config)).expect("yaml"); - let mut file = Mount::new("grid-refused-serving", &[("serving.yaml", &yaml(&serving))]); - let path = file.path("serving.yaml"); + serving.peers[0].grid_ca_path = file("ca.pem"); + serving.peers[0].client_cert_path = file("tls.crt"); + serving.peers[0].client_key_path = file("tls.key"); + let path = dir.join("serving.yaml"); + let write = |config: &GridServingConfig| { + std::fs::write(&path, serde_yaml::to_string(&serde_yaml_value(config)).expect("yaml")).expect("write"); + }; + write(&serving); let peers = Peers::default(); let mut grid = runtime(&peers, &serving); @@ -1275,7 +1231,7 @@ mod tests { let mut twice = config(&["east", "west"]); twice.peers[1].site = "east".to_owned(); - file.write(&[("serving.yaml", &yaml(&twice))]); + write(&twice); eventually("the duplicate site refused", || counts().rejected() == 1); std::thread::sleep(Duration::from_millis(200)); assert_eq!( @@ -1285,7 +1241,9 @@ mod tests { ); assert_eq!(peers.starts("east"), 1, "a refused file starts and drops no poller"); - identity.write(&[("ca.pem", "old"), ("tls.crt", "renewed"), ("tls.key", "renewed")]); + for name in ["tls.crt", "tls.key"] { + std::fs::write(file(name), "renewed").expect("renew"); + } eventually("the running config renewed", || counts().applied() == 1); assert_eq!( peers.starts("east"), @@ -1299,6 +1257,7 @@ mod tests { ); drop(grid); + std::fs::remove_dir_all(&dir).expect("cleanup"); } /// `config` in the operator's serving-config shape. @@ -1307,11 +1266,15 @@ mod tests { .peers .iter() .map(|p| { - serde_yaml::from_str(&format!( + let mut peer: serde_yaml::Value = serde_yaml::from_str(&format!( "{{site: {}, addr: '{}', server_name: {}, authority: {}, grid_ca_path: {}, client_cert_path: {}, client_key_path: {}}}", p.site, p.addr, p.server_name, p.authority, p.grid_ca_path, p.client_cert_path, p.client_key_path )) - .expect("peer") + .expect("peer"); + if let (Some(gateway), Some(fields)) = (&p.gateway, peer.as_mapping_mut()) { + fields.insert("gateway".into(), gateway.clone().into()); + } + peer }) .collect(); let candidates: Vec = config @@ -1333,4 +1296,61 @@ mod tests { root.insert("peers".into(), peers.into()); root.into() } + + /// A writer the test subscriber logs into. + #[derive(Clone, Default)] + struct Captured(Arc>>); + + impl std::io::Write for Captured { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().expect("log buffer").extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + impl<'writer> tracing_subscriber::fmt::MakeWriter<'writer> for Captured { + type Writer = Self; + + fn make_writer(&'writer self) -> Self::Writer { + self.clone() + } + } + + /// What `run` logs at info and above. + fn info_log(run: impl FnOnce()) -> String { + let out = Captured::default(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .with_writer(out.clone()) + .finish(); + tracing::subscriber::with_default(subscriber, run); + String::from_utf8(out.0.lock().expect("log buffer").clone()).expect("utf-8 log") + } + + #[test] + fn unchanged_reload_logs_nothing_at_info() { + let tally = WatchCounts::default(); + let quiet = info_log(|| report(&Ok(None), &tally, "grid: serving config reloaded")); + assert!(quiet.is_empty(), "a no-op reload logged at info: {quiet}"); + assert_eq!( + tally.reused.load(Ordering::SeqCst), + 1, + "the no-op reload is still counted" + ); + + let outcome = ReloadOutcome { + started: 1, + ..ReloadOutcome::default() + }; + let loud = info_log(|| report(&Ok(Some(outcome)), &tally, "grid: serving config reloaded")); + assert!( + loud.contains("grid: serving config reloaded"), + "a real reload logs at info: {loud}" + ); + } } diff --git a/gateway/ai-grid-filters/src/decisions.rs b/gateway/ai-grid-filters/src/decisions.rs new file mode 100644 index 000000000..556c74325 --- /dev/null +++ b/gateway/ai-grid-filters/src/decisions.rs @@ -0,0 +1,133 @@ +//! `grid_route_decisions_total`: where `grid_site_route` sent a request, or why it refused. +//! +//! Counter handles are registered off the request path, so recording is one atomic add. + +use std::sync::{Arc, LazyLock}; + +use metrics::Counter; + +/// The counter's name. +const METRIC: &str = "grid_route_decisions_total"; + +/// The `reason` label: a closed set, so the series count is fixed. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Outcome { + /// Sent to a healthy site. How it ranked is in `grid_route_site_score`. + Routed, + /// Sent to a demoted site because no healthy one was left. + Fallback, + /// Every candidate was excluded. Answered 503. + NotReady, + /// An admitted candidate had no route from this gateway. Answered 503. + NoRoute, + /// The request named no model, or one no candidate serves. Answered 400 or 404. + BadRequest, + /// Every healthy site serving the model was past full. Answered 503. + Shed, +} + +impl Outcome { + /// Every value, in label order. + #[cfg(test)] + pub(crate) const ALL: [Self; 6] = [ + Self::Routed, + Self::Fallback, + Self::NotReady, + Self::NoRoute, + Self::BadRequest, + Self::Shed, + ]; + + /// The `reason` label value. + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Routed => "routed", + Self::Fallback => "fallback", + Self::NotReady => "not_ready", + Self::NoRoute => "no_route", + Self::BadRequest => "bad_request", + Self::Shed => "shed", + } + } +} + +/// Why a request went nowhere. It picks the status, and its [`Outcome`] is the label. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Refused { + /// Every match was excluded, by readiness or admission. + NotReady, + /// An admitted match existed but had no route from this gateway. + NoRoute, + /// No candidate serves the model. + UnknownModel, + /// The request named no model. + NoModel, + /// Every healthy site serving the model was past full. + Shed, +} + +impl Refused { + /// The label this refusal counts under. + pub(crate) fn outcome(self) -> Outcome { + match self { + Self::NotReady => Outcome::NotReady, + Self::NoRoute => Outcome::NoRoute, + Self::UnknownModel | Self::NoModel => Outcome::BadRequest, + Self::Shed => Outcome::Shed, + } + } + + /// Count one refused request, under site "". + pub(crate) fn record(self) { + // Built on first use, after the gateway installs the recorder. + static REFUSED: LazyLock<[Counter; 4]> = LazyLock::new(|| { + [Outcome::NotReady, Outcome::NoRoute, Outcome::BadRequest, Outcome::Shed] + .map(|outcome| metrics::counter!(METRIC, "site" => "", "reason" => outcome.as_str())) + }); + let index = match self.outcome() { + Outcome::NotReady => 0, + Outcome::NoRoute => 1, + Outcome::BadRequest | Outcome::Routed | Outcome::Fallback => 2, + Outcome::Shed => 3, + }; + if let Some(counter) = REFUSED.get(index) { + counter.increment(1); + } + } +} + +/// One site's counters: routed, then fallback. +#[derive(Debug)] +pub struct SiteDecisions([Counter; 2]); + +impl SiteDecisions { + /// Register the counters for `site`, a name from the serving config. + pub(crate) fn new(site: &Arc) -> Self { + Self( + [Outcome::Routed, Outcome::Fallback] + .map(|outcome| metrics::counter!(METRIC, "site" => Arc::clone(site), "reason" => outcome.as_str())), + ) + } + + /// Count one request sent to this site, as a fallback when no healthy site was left. + pub(crate) fn record(&self, fallback: bool) { + if let Some(counter) = self.0.get(usize::from(fallback)) { + counter.increment(1); + } + } +} + +#[cfg(test)] +mod tests { + use super::Outcome; + + #[test] + fn the_reason_label_is_a_closed_set() { + let labels = Outcome::ALL.map(Outcome::as_str); + assert_eq!( + labels, + ["routed", "fallback", "not_ready", "no_route", "bad_request", "shed"], + "adding a reason is a deliberate change: update the chart README metrics section" + ); + } +} diff --git a/gateway/ai-grid-filters/src/descriptor.rs b/gateway/ai-grid-filters/src/descriptor.rs index 73972938d..b7de80f4e 100644 --- a/gateway/ai-grid-filters/src/descriptor.rs +++ b/gateway/ai-grid-filters/src/descriptor.rs @@ -7,6 +7,7 @@ use std::{collections::HashSet, sync::Arc}; +use praxis_core::connectivity::Upstream; use praxis_filter::FilterError; use serde::Deserialize; @@ -50,7 +51,8 @@ impl CapabilityKind { /// /// Controls whether a candidate accepts new sessions, existing sessions only, /// or is excluded from routing entirely. -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] pub enum AdmissionState { /// Accepts both new and existing sessions. #[default] @@ -59,7 +61,8 @@ pub enum AdmissionState { /// Accepts only existing sessions (bound via session affinity). ExistingOnly, - /// Excluded from routing entirely. + /// Excluded from routing entirely. `none` on the wire, as the operator writes it. + #[serde(rename = "none")] Excluded, } @@ -86,6 +89,10 @@ impl AdmissionState { #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] #[serde(deny_unknown_fields)] pub struct CandidateConfig { + /// Whether it takes new requests, as the operator resolved it. Absent means it does. + #[serde(default)] + pub admission: AdmissionState, + /// Cluster name to select when this candidate is chosen. pub cluster: String, @@ -112,19 +119,43 @@ fn default_fresh() -> bool { true } +/// A provider's recent latency, as its site's operator publishes it. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Latency { + /// Median streaming time to first token, seconds. + pub ttft_p50: Option, + /// 90th percentile streaming time to first token, seconds. + pub ttft_p90: Option, + /// Mean streaming time per output token, seconds. + pub tpot: Option, + /// Estimated prefill seconds per uncached input token. + pub prefill_per_token: Option, +} + /// A validated route candidate ready for runtime matching. /// /// Created by `validate_candidates` from raw config entries. All string /// fields are bounded and non-blank. The Grid-owned fields (`admission_state`, /// `rank`, `selection_tier`) are populated after validation from live signals. /// -/// `Clone` is cheap: every owned field is an `Arc`. The refresh step clones +/// `Clone` is cheap: every owned field is reference counted. The refresh step clones /// the base set each poll cycle to re-order it by live load. #[derive(Clone, Debug)] pub struct RouteCandidate { /// Grid-operator admission state. pub admission_state: AdmissionState, + /// Requests this candidate runs at once without engine queueing, as its operator publishes + /// it; `None` when unpublished, which leaves its load unknown. + pub capacity: Option, + + /// Requests held over capacity, as its operator publishes it; `None` when unpublished or + /// stale. Resolved when the snapshot is built, never per request. + pub rho: Option, + + /// Recent latency its operator publishes, each `None` when unpublished or stale. + pub latency: Latency, + /// Cluster name to select. pub cluster: Arc, @@ -152,6 +183,9 @@ pub struct RouteCandidate { /// Deterministic identifier for session affinity binding. pub stable_id: Arc, + + /// Remote site's gateway, dialed directly instead of a `load_balancer` cluster. + pub upstream: Option, } /// Build a deterministic stable ID from candidate identity fields. @@ -200,7 +234,10 @@ pub(crate) fn validate_candidates(raw: Vec) -> Result) -> Result CandidateConfig { let kind: CapabilityKind = serde_yaml::from_str(&format!("\"{kind_str}\"")).unwrap(); CandidateConfig { + admission: AdmissionState::default(), cluster: cluster.to_owned(), credential: None, fresh: true, @@ -384,4 +423,20 @@ mod tests { let err = validate_local_site("").expect_err("should fail"); assert!(err.to_string().contains("local_site must be"), "{err}"); } + + #[test] + fn admission_parses_as_the_operator_writes_it_and_defaults_to_admitted() { + let parse = |json: &str| serde_yaml::from_str::(json).map(|c| c.admission); + let base = r#""kind":"inference_model","name":"m","site":"s","cluster":"c""#; + assert_eq!(parse(&format!("{{{base}}}")).unwrap(), AdmissionState::NewAndExisting); + assert_eq!( + parse(&format!(r#"{{{base},"admission":"none"}}"#)).unwrap(), + AdmissionState::Excluded + ); + assert_eq!( + parse(&format!(r#"{{{base},"admission":"existing_only"}}"#)).unwrap(), + AdmissionState::ExistingOnly + ); + parse(&format!(r#"{{{base},"admission":"maybe"}}"#)).expect_err("an unknown admission is refused"); + } } diff --git a/gateway/ai-grid-filters/src/flow.rs b/gateway/ai-grid-filters/src/flow.rs index 5b7f13733..f238d3e57 100644 --- a/gateway/ai-grid-filters/src/flow.rs +++ b/gateway/ai-grid-filters/src/flow.rs @@ -2,7 +2,7 @@ //! //! The pieces have unit tests in their own crates. These stitch the real //! `PeerScraper`, the `LoadStore`, and `RouteSnapshot::from_store` plus -//! `select_admitted` into one path so the composition is exercised, including +//! `select_spread` into one path so the composition is exercised, including //! the failure cases where a peer is unreachable or untrusted and the router //! must still produce a sound decision. @@ -33,7 +33,7 @@ use tokio_rustls::TlsAcceptor; use crate::{ descriptor::{CandidateConfig, CapabilityKind, RouteCandidate, validate_candidates}, - route::select_admitted, + route::{KeepAll, select_spread}, snapshot::{LOAD_METRIC, RouteSnapshot}, }; @@ -169,6 +169,7 @@ fn candidates(model: &str, sites: &[(&str, &str)]) -> Vec { let raw = sites .iter() .map(|(site, cluster)| CandidateConfig { + admission: crate::descriptor::AdmissionState::default(), cluster: (*cluster).to_owned(), credential: None, fresh: true, @@ -216,7 +217,16 @@ async fn scrape_store_route_picks_the_least_loaded_site() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-b", "the idle site wins end to end"); } @@ -258,7 +268,16 @@ async fn the_scrape_hop_is_mutually_authenticated_then_routes() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-a"); } @@ -314,7 +333,16 @@ async fn loss_of_signal_sorts_the_unmeasured_site_last() { ); // Even though east is busy (90), west has no signal at all, so it sorts // last and the measured site is chosen. Loss of signal is "least preferred". - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!( &*chosen.cluster, "pool-a", "a measured busy site beats an unmeasured one" @@ -348,7 +376,16 @@ async fn an_unreachable_peer_leaves_no_reading_and_the_reachable_one_routes() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-b", "the reachable site is chosen"); } @@ -385,7 +422,16 @@ async fn an_untrusted_peer_is_refused_and_contributes_no_reading() { 1_000, 30_000, ); - let chosen = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, "llama").expect("a route"); + let chosen = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + 0, + |_| true, + &KeepAll, + ) + .expect("a route") + .candidate; assert_eq!(&*chosen.cluster, "pool-b", "only the trusted, measured peer routes"); } diff --git a/gateway/ai-grid-filters/src/health.rs b/gateway/ai-grid-filters/src/health.rs new file mode 100644 index 000000000..5ba8df73f --- /dev/null +++ b/gateway/ai-grid-filters/src/health.rs @@ -0,0 +1,138 @@ +//! Which backend clusters have no healthy endpoint, resolved off the request path. +//! +//! The route filter publishes Praxis's health registry here; the control step reads +//! it on its own tick, so a request never judges raw health. + +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, +}; + +use arc_swap::{ArcSwap, ArcSwapOption}; +use praxis_core::health::{ClusterHealthState, HealthRegistry}; + +/// Clusters whose every endpoint is down, kept by the control step. +#[derive(Debug, Default)] +pub struct ClusterHealth { + /// The pipeline's registry, published by the route filter. + registry: ArcSwapOption, ClusterHealthState>>, + /// The clusters with no healthy endpoint as of the last tick. + down: ArcSwap>>, +} + +impl ClusterHealth { + /// Publish `registry` when it is not the one already held: one lock-free load per request. + pub(crate) fn observe(&self, registry: Option<&HealthRegistry>) { + let Some(registry) = registry else { + return; + }; + let held = self.registry.load(); + if !held.as_ref().is_some_and(|held| Arc::ptr_eq(held, registry)) { + self.registry.store(Some(Arc::clone(registry))); + } + } + + /// Whether praxis has reported any cluster health yet. + /// + /// Without a registry the gateway knows nothing about backend health, so demotion + /// stays off rather than treating every cluster as healthy or as down. + pub(crate) fn observed(&self) -> bool { + self.registry.load().is_some() + } + + /// The clusters that were down as of the last [`Self::update`]. + pub(crate) fn down(&self) -> Arc>> { + self.down.load_full() + } + + /// Recompute the down set from the registry, returning whether it changed. + pub(crate) fn update(&self) -> bool { + let now = self.registry.load().as_deref().map(down_clusters).unwrap_or_default(); + if **self.down.load() == now { + return false; + } + tracing::info!(down = ?now, "grid: backend clusters with no healthy endpoint changed"); + self.down.store(Arc::new(now)); + true + } +} + +/// Clusters in `registry` with endpoints, none of them healthy. +fn down_clusters(registry: &HashMap, ClusterHealthState>) -> BTreeSet> { + registry + .iter() + .filter(|(_, state)| { + let (healthy, total) = state.endpoint_counts(); + total > 0 && healthy == 0 + }) + .map(|(name, _)| Arc::clone(name)) + .collect() +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing, reason = "tests")] +mod tests { + use praxis_core::health::{ClusterHealthEntry, EndpointHealth}; + + use super::*; + + fn registry(clusters: &[(&str, &[bool])]) -> HealthRegistry { + Arc::new( + clusters + .iter() + .map(|(name, endpoints)| { + let health: Vec = endpoints + .iter() + .map(|healthy| { + let endpoint = EndpointHealth::new(); + if !healthy { + endpoint.mark_unhealthy(); + } + endpoint + }) + .collect(); + let addresses = (0..endpoints.len()) + .map(|i| Arc::from(format!("10.0.0.{i}:80"))) + .collect(); + let entry = ClusterHealthEntry::new(health, addresses, None, None); + (Arc::from(*name), Arc::new(entry)) + }) + .collect(), + ) + } + + #[test] + fn a_cluster_is_down_only_when_every_endpoint_is() { + let health = ClusterHealth::default(); + health.observe(Some(®istry(&[ + ("site-a", &[false]), + ("site-b", &[false, true]), + ("site-d", &[true]), + ]))); + assert!(health.update(), "the first tick finds site-a down"); + assert_eq!(*health.down(), BTreeSet::from([Arc::from("site-a")])); + assert!(!health.update(), "an unchanged registry is not news"); + } + + #[test] + fn a_recovered_cluster_leaves_the_down_set() { + let health = ClusterHealth::default(); + health.observe(Some(®istry(&[("site-a", &[false])]))); + assert!(health.update()); + health.observe(Some(®istry(&[("site-a", &[true])]))); + assert!(health.update(), "the recovered cluster leaves the down set"); + assert!(health.down().is_empty()); + } + + #[test] + fn nothing_is_down_and_nothing_is_observed_without_a_registry() { + let health = ClusterHealth::default(); + health.observe(None); + assert!(!health.observed(), "no registry, so demotion stays off"); + assert!(!health.update()); + assert!(health.down().is_empty()); + health.observe(Some(®istry(&[("site-a", &[true])]))); + assert!(health.observed(), "praxis reported, so demotion applies"); + } +} diff --git a/gateway/ai-grid-filters/src/lib.rs b/gateway/ai-grid-filters/src/lib.rs index e723b5f56..458692e52 100644 --- a/gateway/ai-grid-filters/src/lib.rs +++ b/gateway/ai-grid-filters/src/lib.rs @@ -6,9 +6,11 @@ //! contribution is ordering the candidates by live load off the request path. mod control; +mod decisions; mod descriptor; #[cfg(test)] mod flow; +mod health; mod metadata; mod route; mod serving; @@ -18,10 +20,12 @@ use std::sync::Arc; use arc_swap::ArcSwap; pub use control::ReloadOutcome; +pub use decisions::SiteDecisions; // The routing model and the snapshot builder are the crate's control-plane API: // the gateway's refresh step orders candidates by live load and swaps the // snapshot. The request path only reads a snapshot. -pub use descriptor::{AdmissionState, CandidateConfig, CapabilityKind, RouteCandidate}; +pub use descriptor::{AdmissionState, CandidateConfig, CapabilityKind, Latency, RouteCandidate}; +pub use health::ClusterHealth; pub use metadata::{CandidateCredential, CredentialRef}; use praxis_filter::{FilterError, FilterFactory, FilterRegistry, HttpFilter}; pub use serving::{GridRuntime, GridServingConfig, PeerServingConfig, load_serving_config, spawn_grid_routing}; @@ -31,7 +35,7 @@ pub use snapshot::RouteSnapshot; /// owns and its refresh loop swaps. /// /// Call this from the gateway after `FilterRegistry::with_builtins()`, passing -/// the snapshot from [`spawn_grid_routing`]. The factory captures the snapshot, +/// the snapshot and cluster health from [`spawn_grid_routing`]. The factory captures the snapshot, /// so every filter praxis rebuilds on a config reload clones the same `Arc` and /// sees the live swaps. /// @@ -41,9 +45,10 @@ pub use snapshot::RouteSnapshot; pub fn register_grid_filters( registry: &mut FilterRegistry, snapshot: Arc>, + health: Arc, ) -> Result<(), FilterError> { let factory = move |config: &serde_yaml::Value| -> Result, FilterError> { - route::GridSiteRouteFilter::from_config(config, Arc::clone(&snapshot)) + route::GridSiteRouteFilter::from_config(config, Arc::clone(&snapshot), Arc::clone(&health)) }; registry.register("grid_site_route", FilterFactory::Http(Arc::new(factory))) } diff --git a/gateway/ai-grid-filters/src/route.rs b/gateway/ai-grid-filters/src/route.rs index 6b18e852f..54e1822fb 100644 --- a/gateway/ai-grid-filters/src/route.rs +++ b/gateway/ai-grid-filters/src/route.rs @@ -1,19 +1,31 @@ //! The `grid_site_route` filter: pick a cross-site cluster for a request. //! -//! Reads the model header, finds the front admitted candidate for that model in -//! the current snapshot, and sets `ctx.cluster` for the downstream load -//! balancer. Selection is `select_admitted` over a pre-ordered list. The -//! ordering by live load happens off the request path in `snapshot`. +//! Reads the model header, chooses a site for that model from the current snapshot +//! by what each provider publishes, and sets `ctx.cluster` for the downstream load +//! balancer. Each site's load, requests held over capacity (rho), is resolved when +//! the snapshot is built, so a request only filters and draws. A request naming no +//! model gets 400, one for an unknown model 404, and one with no healthy site or a +//! shed model 503 with Retry-After and an OpenAI-style error, logged at debug. Every outcome counts in +//! `grid_route_decisions_total` by site and reason. A routed response names the +//! chosen site and cluster in `x-grid-site` and `x-grid-backend`. -use std::sync::Arc; +use std::{ + collections::HashSet, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, +}; use arc_swap::ArcSwap; use async_trait::async_trait; -use praxis_filter::{FilterAction, FilterError, HttpFilter, HttpFilterContext, Rejection, parse_filter_config}; +use praxis_filter::{FilterAction, FilterError, HttpFilter, HttpFilterContext, TerminalResponse, parse_filter_config}; use serde::Deserialize; use crate::{ + decisions::{Refused, SiteDecisions}, descriptor::{AdmissionState, CapabilityKind, RouteCandidate, validate_model_header}, + health::ClusterHealth, snapshot::RouteSnapshot, }; @@ -33,6 +45,90 @@ struct GridSiteRouteConfig { /// Request header naming the model (default `X-Model`). #[serde(default = "default_model_header")] model_header: String, + + /// The `load_balancer` clusters this gateway declares. When set, a candidate + /// with no peer gateway and a cluster not listed has no route and is skipped. + #[serde(default)] + clusters: Option>, +} + +/// How a cluster's endpoints are dialed, as its `load_balancer` cluster declares. +/// +/// Parsed and validated but unused: nothing probes endpoints in this filter yet, and a +/// config carrying a transport must still load rather than fail the gateway's start. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +enum TransportKind { + /// Grid mutual TLS with the site identity. + MutualTls, + /// Server-only TLS, verified with the cluster's CA. + Tls, + /// No TLS. + Plaintext, +} + +/// One declared cluster: a bare name, or a name with the transport its endpoints are dialed over. +#[derive(Debug)] +enum DeclaredCluster { + /// Routable, not probed. + Name(String), + /// Routable, and carrying the transport its endpoints are dialed over. + Probed(ProbedCluster), +} + +// By hand, not untagged: untagged hides which field or transport was wrong. +impl<'de> Deserialize<'de> for DeclaredCluster { + fn deserialize>(deserializer: D) -> Result { + struct Visitor; + + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = DeclaredCluster; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a cluster name or {name, transport}") + } + + fn visit_str(self, v: &str) -> Result { + Ok(DeclaredCluster::Name(v.to_owned())) + } + + fn visit_map>(self, map: A) -> Result { + ProbedCluster::deserialize(serde::de::value::MapAccessDeserializer::new(map)) + .map(DeclaredCluster::Probed) + } + } + + deserializer.deserialize_any(Visitor) + } +} + +/// A declared cluster that names how its remote endpoints are dialed. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +#[expect( + dead_code, + reason = "parsed so a config carrying a transport loads; nothing probes yet" +)] +struct ProbedCluster { + /// The `load_balancer` cluster name. + name: String, + /// How the cluster dials its endpoints. + transport: TransportKind, + /// The CA a `tls` cluster verifies with. + #[serde(default)] + ca_path: Option, + /// The server name a `tls` cluster expects. + #[serde(default)] + sni: Option, +} + +impl DeclaredCluster { + /// The cluster name. + fn name(&self) -> &str { + match self { + Self::Name(name) | Self::Probed(ProbedCluster { name, .. }) => name, + } + } } /// Routes a request to a cross-site cluster by model, honouring live-load order. @@ -44,6 +140,15 @@ pub(crate) struct GridSiteRouteFilter { /// Header the request carries the model name in. model_header: http::header::HeaderName, + + /// Counts requests from a random start, so gateway replicas draw independently. + turn: AtomicUsize, + + /// Where the filter publishes Praxis's health registry for the control step. + health: Arc, + + /// Declared clusters, `None` when the config does not list them. + clusters: Option>>, } impl GridSiteRouteFilter { @@ -58,10 +163,40 @@ impl GridSiteRouteFilter { pub(crate) fn from_config( config: &serde_yaml::Value, snapshot: Arc>, + health: Arc, ) -> Result, FilterError> { let cfg: GridSiteRouteConfig = parse_filter_config("grid_site_route", config)?; let model_header = validate_model_header(&cfg.model_header)?; - Ok(Box::new(Self { snapshot, model_header })) + let declared = cfg.clusters; + tracing::info!( + "grid_site_route: choosing sites by published in-flight per capacity: two choices among 3 or more \ + sites with room, a weighted draw between 2, and a capacity draw when none has room or none publishes" + ); + Ok(Box::new(Self { + snapshot, + model_header, + turn: AtomicUsize::new(random_seed()), + health, + clusters: declared.map(|clusters| clusters.iter().map(|cluster| Arc::from(cluster.name())).collect()), + })) + } + + /// The site for a request for `model`, or the answer when no site can take it. + fn choose<'snap>(&self, snapshot: &'snap RouteSnapshot, model: &str) -> Result, FilterAction> { + let turn = self.turn.fetch_add(1, Ordering::Relaxed); + if snapshot.shedding.contains(model) { + return Err(refuse(Refused::Shed, model, turn)); + } + let routable = |candidate: &RouteCandidate| has_route(candidate, self.clusters.as_ref()); + select_spread( + snapshot, + CapabilityKind::InferenceModel, + model, + turn, + routable, + &KeepAll, + ) + .ok_or_else(|| refuse(unserved(snapshot, model, routable), model, turn)) } } @@ -76,8 +211,9 @@ impl HttpFilter for GridSiteRouteFilter { } async fn on_request(&self, ctx: &mut HttpFilterContext<'_>) -> Result { - // An earlier cluster-selecting filter wins; never override its choice. - if ctx.cluster.is_some() { + self.health.observe(ctx.health_registry); + // An earlier selecting filter wins; never override its choice. + if ctx.cluster.is_some() || ctx.upstream.is_some() { return Ok(FilterAction::Continue); } // Borrow the model header out of *ctx.request; that field is disjoint @@ -88,35 +224,416 @@ impl HttpFilter for GridSiteRouteFilter { .get(&self.model_header) .and_then(|value| value.to_str().ok()) else { - // No model header: not ours to route, leave it for the next filter. - return Ok(FilterAction::Continue); + // Nothing after this filter selects a cluster, so answer here rather than fail in the load balancer. + tracing::debug!(path = %ctx.request.uri.path(), "grid_site_route: no model in the request"); + Refused::NoModel.record(); + return Ok(unrouted(400)); }; let snapshot = self.snapshot.load(); - let Some(candidate) = select_admitted(&snapshot.candidates, CapabilityKind::InferenceModel, model) else { - tracing::debug!(model = %model, "grid_site_route: no admitted candidate"); - return Ok(FilterAction::Reject(Rejection::status(404))); + match self.choose(&snapshot, model) { + Ok(pick) => { + route_to(ctx, &pick); + Ok(FilterAction::Continue) + }, + Err(answer) => Ok(answer), + } + } +} + +/// Send the request to `pick`: count the decision and select its cluster. +fn route_to(ctx: &mut HttpFilterContext<'_>, pick: &Pick<'_>) { + pick.decisions.record(pick.fallback); + ctx.cluster = Some(Arc::clone(&pick.candidate.cluster)); +} + +/// Count and answer a request no candidate took. +fn refuse(reason: Refused, model: &str, turn: usize) -> FilterAction { + tracing::debug!(model = %model, reason = ?reason, "grid_site_route: no admitted candidate"); + reason.record(); + // Known but excluded everywhere is a temporary outage, not an unknown model. + match reason { + Refused::UnknownModel => error_response( + 404, + "invalid_request_error", + "model_not_found", + "no site serves this model", + None, + ), + // Load, not an outage: 429 is what an OpenAI client backs off and retries on. + Refused::Shed => retry_later( + turn, + 429, + "rate_limit_exceeded", + "capacity_exhausted", + "every site serving this model is at capacity", + ), + Refused::NotReady | Refused::NoRoute | Refused::NoModel => retry_later( + turn, + 503, + "server_error", + "no_healthy_site", + "no healthy site serves this model now", + ), + } +} + +/// Why no candidate took a request for `model`: unknown, unroutable, or excluded. +fn unserved(snapshot: &RouteSnapshot, model: &str, routable: impl Fn(&RouteCandidate) -> bool) -> Refused { + let mut matches = snapshot + .candidates + .iter() + .filter(|candidate| candidate.kind == CapabilityKind::InferenceModel && &*candidate.name == model) + .peekable(); + if matches.peek().is_none() { + return Refused::UnknownModel; + } + if matches.any(|candidate| is_admitted_for_new_request(candidate.admission_state) && !routable(candidate)) { + Refused::NoRoute + } else { + Refused::NotReady + } +} + +/// The candidate `select_spread` chose and why. +#[derive(Debug)] +pub(crate) struct Pick<'snap> { + /// The chosen candidate. + pub(crate) candidate: &'snap RouteCandidate, + /// Whether it was demoted, chosen only because no healthy site was left. + pub(crate) fallback: bool, + /// The chosen site's decision counters. + pub(crate) decisions: &'snap SiteDecisions, +} + +impl<'snap> Pick<'snap> { + /// `site` as chosen, a `fallback` when no healthy site was left. + const fn new((candidate, decisions): Site<'snap>, fallback: bool) -> Self { + Self { + candidate, + fallback, + decisions, + } + } +} + +/// The router's own answer to a request it cannot route: a response, not a rejection, +/// since praxis logs every rejection at WARN. +fn unrouted(status: u16) -> FilterAction { + FilterAction::TerminalResponse(Box::new(TerminalResponse::new(status))) +} + +/// Seconds a client should wait before retrying a model with no admitted candidate: +/// about one operator scrape plus one peer poll, spread so retries do not arrive together. +const RETRY_AFTER_SECS: [&str; 5] = ["3", "4", "5", "6", "7"]; + +/// `status` with a Retry-After picked by `turn` and an OpenAI-style error naming `code`. +fn retry_later(turn: usize, status: u16, kind: &str, code: &str, message: &str) -> FilterAction { + let retry_after = turn + .checked_rem(RETRY_AFTER_SECS.len()) + .and_then(|index| RETRY_AFTER_SECS.get(index)) + .copied() + .unwrap_or("5"); + error_response(status, kind, code, message, Some(retry_after)) +} + +/// A gateway-originated error: `status`, an OpenAI-style error object, and an optional Retry-After. +fn error_response( + status: u16, + kind: &str, + code: &str, + message: &str, + retry_after: Option<&'static str>, +) -> FilterAction { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + if let Some(retry_after) = retry_after { + headers.insert(http::header::RETRY_AFTER, http::HeaderValue::from_static(retry_after)); + } + // Every code and message is a constant here, so none needs JSON escaping. + let body = format!(r#"{{"error":{{"message":"{message}","type":"{kind}","code":"{code}"}}}}"#); + FilterAction::TerminalResponse(Box::new( + TerminalResponse::new(status).with_headers(headers).with_body(body), + )) +} + +/// Whether a request routed to `candidate` has somewhere to go: a declared cluster. +/// With no declared list, every cluster is assumed to exist. +fn has_route(candidate: &RouteCandidate, clusters: Option<&HashSet>>) -> bool { + clusters.is_none_or(|clusters| clusters.contains(&candidate.cluster)) +} + +/// One site the draw may choose, as a [`Narrow`] stage sees it. +#[derive(Clone, Copy, Debug)] +#[cfg_attr( + not(test), + expect(dead_code, reason = "read by narrowing stages; this crate ships only KeepAll") +)] +pub(crate) struct SiteView<'snap> { + /// The candidate. + pub(crate) candidate: &'snap RouteCandidate, + /// Requests it holds over its capacity, below 1 for every site the draw sees. + pub(crate) rho: f64, + /// Expected delay in seconds; `None` until ranking uses latency. + pub(crate) delay: Option, + /// Recent latency its operator publishes, each `None` until enough requests complete. + pub(crate) latency: crate::descriptor::Latency, +} + +/// A per-request stage that narrows the sites with room before the draw. +/// +/// It may only clear entries of `keep`, which start all true, so it never adds a site +/// that health, readiness, or shedding removed. Clearing every entry keeps them all. +pub(crate) trait Narrow { + /// Clear `keep[i]` for each of `sites` the draw should not consider. + fn keep(&self, sites: &[SiteView<'_>], keep: &mut [bool]); +} + +/// Keeps every site: selection by load alone. +pub(crate) struct KeepAll; + +impl Narrow for KeepAll { + fn keep(&self, _sites: &[SiteView<'_>], _keep: &mut [bool]) {} +} + +/// Most sites with room one request weighs. Past it, the rest wait for a later snapshot order. +const MAX_SITES: usize = 64; + +/// Bounds of either site's share when two sites with room are drawn between. +const TWO_SITE_SHARE: (f64, f64) = (0.1, 0.9); + +/// A candidate with its decision counters. +type Site<'snap> = (&'snap RouteCandidate, &'snap SiteDecisions); + +/// Up to [`MAX_SITES`] sites, the first `len` filled. +struct Sites<'snap> { + /// The sites, filled from the front. + slots: [Option>; MAX_SITES], + /// How many slots are filled. + len: usize, +} + +impl<'snap> Sites<'snap> { + /// The first [`MAX_SITES`] of `sites`, warning once when more are offered. + fn gather(sites: impl Iterator>) -> Self { + let mut gathered = Self { + slots: [None; MAX_SITES], + len: 0, }; - ctx.cluster = Some(Arc::clone(&candidate.cluster)); - Ok(FilterAction::Continue) + for site in sites { + let Some(slot) = gathered.slots.get_mut(gathered.len) else { + warn_past_max_sites(); + break; + }; + *slot = Some(site); + gathered.len = gathered.len.saturating_add(1); + } + gathered + } + + /// The filled sites in order. + fn iter(&self) -> impl Iterator> + '_ { + self.slots.iter().take(self.len).flatten().copied() + } + + /// The site at `index`, if filled. + fn get(&self, index: usize) -> Option> { + self.slots.get(..self.len)?.get(index).copied().flatten() } } -/// The front candidate matching `kind` and `name` that admits new requests. +/// Choose a site for `name` among the admitted, `routable` matches. /// -/// The list is pre-ordered (least-loaded first), so the front match is the -/// chosen target. This is the reused selection: a linear first-match, never a -/// score computed here. -pub(crate) fn select_admitted<'list>( - candidates: &'list [RouteCandidate], +/// Among healthy matches with room (rho below 1) that `narrow` keeps: two choices at +/// three or more, drawn by capacity, taking the lower rho; a draw between two, weighted +/// by capacity over 1 + rho with each share clamped to 0.1..0.9; or the only one. With +/// none, a capacity draw over the healthy matches tied on the best polled queue score (a +/// prefix, since scores ascend), which covers sites that publish no rho without herding. With no healthy +/// match, the front demoted one, as a fallback. +#[expect( + clippy::too_many_arguments, + reason = "the request path passes its inputs directly rather than build a struct per request" +)] +pub(crate) fn select_spread<'snap>( + snapshot: &'snap RouteSnapshot, kind: CapabilityKind, name: &str, -) -> Option<&'list RouteCandidate> { - candidates.iter().find(|candidate| { - candidate.kind == kind && &*candidate.name == name && is_admitted_for_new_request(candidate.admission_state) + turn: usize, + routable: impl Fn(&RouteCandidate) -> bool, + narrow: &impl Narrow, +) -> Option> { + let matches = matching(snapshot, kind, name, &routable); + let (front, _, front_decisions) = matches.clone().next()?; + let healthy = matches.filter(|(_, score, _)| !score.is_nan()); + let Some((_, best, _)) = healthy.clone().next() else { + // Demotion scores a candidate NaN: every healthy site is gone, so try the front one. + return Some(Pick::new((front, front_decisions), true)); + }; + let site = |(candidate, _, decisions): Scored<'snap>| (candidate, decisions); + let room = Sites::gather(healthy.clone().filter(|(candidate, ..)| has_room(candidate)).map(site)); + let band = || { + Sites::gather( + healthy + .take_while(|(_, score, _)| score.total_cmp(&best).is_eq()) + .map(site), + ) + }; + draw(&narrowed(&room, narrow), band, turn).map(|chosen| Pick::new(chosen, false)) +} + +/// A candidate with its score and decision counters. +type Scored<'snap> = (&'snap RouteCandidate, f64, &'snap SiteDecisions); + +/// The admitted, `routable` candidates for `kind` and `name`, in snapshot order. +fn matching<'snap, 'req>( + snapshot: &'snap RouteSnapshot, + kind: CapabilityKind, + name: &'req str, + routable: &'req impl Fn(&RouteCandidate) -> bool, +) -> impl Iterator> + Clone + 'req +where + 'snap: 'req, +{ + snapshot + .candidates + .iter() + .zip(&snapshot.scores) + .zip(&snapshot.decisions) + .map(|((candidate, score), decisions)| (candidate, *score, decisions)) + .filter(move |(candidate, ..)| { + candidate.kind == kind + && &*candidate.name == name + && is_admitted_for_new_request(candidate.admission_state) + && routable(candidate) + }) +} + +/// The tier for `room` sites: two choices at three or more, a weighted draw at two, the +/// only one at one, and with none a capacity draw over `band`. +fn draw<'snap>(room: &Sites<'snap>, band: impl FnOnce() -> Sites<'snap>, turn: usize) -> Option> { + match room.len { + 0 => by_capacity(&band(), turn), + 1 => room.get(0), + 2 => Some(between_two(room.get(0)?, room.get(1)?, turn)), + _ => two_choices(room, turn), + } +} + +/// Whether `candidate` publishes load and has room for one more request. +fn has_room(candidate: &RouteCandidate) -> bool { + candidate.rho.is_some_and(|rho| rho < 1.0) +} + +/// The sites `narrow` keeps of `room`, or all of them when it keeps none. +fn narrowed<'snap>(room: &Sites<'snap>, narrow: &impl Narrow) -> Sites<'snap> { + let views: [Option>; MAX_SITES] = std::array::from_fn(|index| { + room.get(index).map(|(candidate, _)| SiteView { + candidate, + rho: candidate.rho.unwrap_or(0.0), + delay: None, + latency: candidate.latency, + }) + }); + let views: Vec> = views.iter().flatten().copied().collect(); + let mut keep = [true; MAX_SITES]; + if let Some(kept) = keep.get_mut(..room.len) { + narrow.keep(&views, kept); + } + if !keep.iter().take(room.len).any(|kept| *kept) { + return Sites::gather(room.iter()); + } + Sites::gather(room.iter().zip(keep).filter(|(_, kept)| *kept).map(|(site, _)| site)) +} + +/// A capacity draw over `sites`. Unpublished capacity weighs the mean of the published, or 1. +fn by_capacity<'snap>(sites: &Sites<'snap>, turn: usize) -> Option> { + let published: Vec = sites.iter().filter_map(|(candidate, _)| candidate.capacity).collect(); + let unpublished = if published.is_empty() { + 1.0 + } else { + published.iter().sum::() / published.iter().map(|_| 1.0).sum::() + }; + weighted(sites, unit(turn, SALTS.0), None, |candidate| { + candidate.capacity.unwrap_or(unpublished) }) } +/// A draw between two sites by capacity over 1 + rho, each share clamped to 0.1..0.9. +fn between_two<'snap>(first: Site<'snap>, second: Site<'snap>, turn: usize) -> Site<'snap> { + let rate = |candidate: &RouteCandidate| candidate.capacity.unwrap_or(1.0) / (1.0 + candidate.rho.unwrap_or(0.0)); + let share = (rate(first.0) / (rate(first.0) + rate(second.0))).clamp(TWO_SITE_SHARE.0, TWO_SITE_SHARE.1); + if unit(turn, SALTS.0) < share { first } else { second } +} + +/// Two distinct draws from `sites` by capacity, taking the lower rho; exact ties go to the first. +fn two_choices<'snap>(sites: &Sites<'snap>, turn: usize) -> Option> { + let capacity = |candidate: &RouteCandidate| candidate.capacity.unwrap_or(1.0); + let first = weighted(sites, unit(turn, SALTS.0), None, capacity)?; + let second = weighted(sites, unit(turn, SALTS.1), Some(first.0), capacity).unwrap_or(first); + Some(if second.0.rho < first.0.rho { second } else { first }) +} + +/// The site `draw` (in 0..1) lands on when `sites` are weighted by `weight`, skipping `skip`. +fn weighted<'snap>( + sites: &Sites<'snap>, + draw: f64, + skip: Option<&RouteCandidate>, + weight: impl Fn(&RouteCandidate) -> f64, +) -> Option> { + let eligible = || { + sites + .iter() + .filter(|(candidate, _)| skip.is_none_or(|skip| !std::ptr::eq(*candidate, skip))) + }; + let mut target = draw * eligible().map(|(candidate, _)| weight(candidate)).sum::(); + let mut last = None; + for site in eligible() { + last = Some(site); + target -= weight(site.0); + if target < 0.0 { + return Some(site); + } + } + last +} + +/// Salts that make a request's two draws independent. +const SALTS: (u64, u64) = (0x9E37_79B9_7F4A_7C15, 0xC2B2_AE3D_27D4_EB4F); + +/// A uniform value in [0, 1) from `turn` and `salt`, mixed so consecutive turns do not correlate. +fn unit(turn: usize, salt: u64) -> f64 { + let mut mixed = u64::try_from(turn).unwrap_or(u64::MAX) ^ salt; + mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + mixed ^= mixed >> 31; + // The top 53 bits as a fraction, built from two exact halves. + let bits = mixed >> 11; + let high = u32::try_from(bits >> 21).unwrap_or(0); + let low = u32::try_from(bits & 0x1F_FFFF).unwrap_or(0); + (f64::from(high) * 2_f64.powi(21) + f64::from(low)) / 2_f64.powi(53) +} + +/// A random starting turn, so replicas started together do not draw in step. +fn random_seed() -> usize { + use std::hash::BuildHasher as _; + let hashed = std::collections::hash_map::RandomState::new().hash_one(std::time::SystemTime::now()); + usize::try_from(hashed).unwrap_or_else(|_| usize::try_from(hashed >> 32).unwrap_or(0)) +} + +/// Log once that a model has more sites than one request weighs. +fn warn_past_max_sites() { + static WARNED: std::sync::Once = std::sync::Once::new(); + WARNED.call_once(|| { + tracing::warn!( + max = MAX_SITES, + "grid_site_route: more sites than one request weighs; the rest wait for a later order" + ); + }); +} + /// Whether a candidate in this admission state accepts a new request. fn is_admitted_for_new_request(state: AdmissionState) -> bool { matches!(state, AdmissionState::NewAndExisting) @@ -130,15 +647,25 @@ fn is_admitted_for_new_request(state: AdmissionState) -> bool { clippy::indexing_slicing, clippy::panic, clippy::min_ident_chars, + clippy::type_complexity, + clippy::arithmetic_side_effects, + clippy::float_arithmetic, reason = "tests" )] mod tests { use super::*; use crate::descriptor::{CandidateConfig, validate_candidates}; + /// The pick for `kind` and `name` over `candidates` in their given order. + fn front(candidates: Vec, kind: CapabilityKind, name: &str) -> Option { + let snapshot = RouteSnapshot::from_static(candidates, Arc::from("east")); + select_spread(&snapshot, kind, name, 0, |_| true, &KeepAll).map(|pick| pick.candidate.cluster.to_string()) + } + /// A validated one-candidate list for `model` at `site`/`cluster`. fn one(model: &str, site: &str, cluster: &str, admission: AdmissionState) -> Vec { let mut candidates = validate_candidates(vec![CandidateConfig { + admission: AdmissionState::default(), cluster: cluster.to_owned(), credential: None, fresh: true, @@ -151,28 +678,417 @@ mod tests { candidates } + /// Three tied candidates for `m`, at sites a, b and d. + fn three() -> Vec { + ["a", "b", "d"] + .iter() + .flat_map(|site| one("m", site, &format!("pool-{site}"), AdmissionState::NewAndExisting)) + .collect() + } + + fn pick(snapshot: &RouteSnapshot, turn: usize) -> String { + select_spread(snapshot, CapabilityKind::InferenceModel, "m", turn, |_| true, &KeepAll) + .map(|chosen| chosen.candidate.cluster.to_string()) + .unwrap() + } + + #[test] + fn tied_candidates_split_evenly() { + let snapshot = RouteSnapshot::from_static(three(), Arc::from("hub")); + let picks: Vec = (0..3_000).map(|turn| pick(&snapshot, turn)).collect(); + for cluster in ["pool-a", "pool-b", "pool-d"] { + let taken = picks.iter().filter(|p| *p == cluster).count(); + assert!((850..=1_150).contains(&taken), "{cluster} took {taken} of 3000"); + } + } + + #[test] + fn a_declared_cluster_is_a_name_or_a_name_with_its_transport() { + let cfg: GridSiteRouteConfig = serde_yaml::from_str( + "clusters:\n - site-a\n - name: east\n transport: tls\n ca_path: /etc/ca.crt\n sni: east.svc\n - name: west\n transport: plaintext\n", + ) + .expect("a config carrying transports still loads"); + let declared = cfg.clusters.expect("declared"); + let names: Vec<&str> = declared.iter().map(DeclaredCluster::name).collect(); + assert_eq!(names, ["site-a", "east", "west"], "every declared cluster is routable"); + let unknown = serde_yaml::from_str::("clusters:\n - name: east\n transport: quic\n") + .expect_err("an unknown transport is rejected"); + assert!(unknown.to_string().contains("quic"), "{unknown}"); + assert!( + serde_yaml::from_str::("clusters:\n - name: east\n transport: tls\n ca: /x\n") + .is_err(), + "an unknown key is rejected" + ); + } + + #[test] + fn a_candidate_with_no_peer_gateway_and_no_declared_cluster_has_no_route() { + let mut candidates = one("llama", "site-d", "site-d", AdmissionState::NewAndExisting); + candidates.extend(one("llama", "dagobah", "site-a", AdmissionState::NewAndExisting)); + let snapshot = RouteSnapshot::from_static(candidates, Arc::from("dagobah")); + let declared: HashSet> = HashSet::from([Arc::from("site-a")]); + for turn in 0..4 { + let picked = select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |c| has_route(c, Some(&declared)), + &KeepAll, + ) + .expect("site-a is routable") + .candidate; + assert_eq!( + &*picked.cluster, "site-a", + "site-d is not a declared cluster, turn {turn}" + ); + } + assert!( + snapshot.candidates.iter().all(|c| has_route(c, None)), + "with no declared list, every cluster is assumed to exist" + ); + let undeclared = one("llama", "site-d", "site-d", AdmissionState::NewAndExisting).remove(0); + assert!( + !has_route(&undeclared, Some(&declared)), + "a cluster the config never declared is no route" + ); + } + + /// Whether the pick at `turn` for `llama` over `snapshot` was a fallback. + fn fallback(snapshot: &RouteSnapshot, turn: usize) -> Option { + select_spread( + snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |_| true, + &KeepAll, + ) + .map(|pick| pick.fallback) + } + + #[test] + fn a_pick_is_a_fallback_only_when_no_healthy_site_is_left() { + assert_eq!(fallback(&two_sites(Some(1.0), Some(5.0)), 0), Some(false)); + assert_eq!(fallback(&two_sites(Some(2.0), Some(2.0)), 1), Some(false)); + let alone = RouteSnapshot::from_static( + one("llama", "east", "pool-a", AdmissionState::NewAndExisting), + Arc::from("east"), + ); + assert_eq!(fallback(&alone, 0), Some(false)); + let down = std::collections::BTreeSet::from([Arc::from("pool-a"), Arc::from("pool-b")]); + assert_eq!(fallback(&two_sites(Some(1.0), Some(5.0)).demote(&down), 0), Some(true)); + // One cluster left healthy is a routed pick, not a fallback. + let one = std::collections::BTreeSet::from([Arc::from("pool-a")]); + assert_eq!(fallback(&two_sites(Some(1.0), Some(5.0)).demote(&one), 0), Some(false)); + } + + #[test] + fn an_unserved_request_names_why() { + let mut candidates = one("llama", "east", "pool-a", AdmissionState::Excluded); + candidates.extend(one("llama", "west", "pool-b", AdmissionState::NewAndExisting)); + let snapshot = RouteSnapshot::from_static(candidates, Arc::from("east")); + assert_eq!(unserved(&snapshot, "granite", |_| true), Refused::UnknownModel); + assert_eq!(unserved(&snapshot, "llama", |_| false), Refused::NoRoute); + let excluded = RouteSnapshot::from_static( + one("llama", "east", "pool-a", AdmissionState::Excluded), + Arc::from("east"), + ); + assert_eq!(unserved(&excluded, "llama", |_| false), Refused::NotReady); + } + #[test] fn front_match_is_selected() { let candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); - let chosen = select_admitted(&candidates, CapabilityKind::InferenceModel, "llama").expect("a match"); - assert_eq!(&*chosen.cluster, "pool-a"); + assert_eq!( + front(candidates, CapabilityKind::InferenceModel, "llama").as_deref(), + Some("pool-a") + ); } #[test] fn a_different_model_does_not_match() { let candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); - assert!(select_admitted(&candidates, CapabilityKind::InferenceModel, "granite").is_none()); + assert!(front(candidates, CapabilityKind::InferenceModel, "granite").is_none()); } #[test] fn an_excluded_candidate_is_skipped() { let candidates = one("llama", "east", "pool-a", AdmissionState::Excluded); - assert!(select_admitted(&candidates, CapabilityKind::InferenceModel, "llama").is_none()); + assert!(front(candidates, CapabilityKind::InferenceModel, "llama").is_none()); } #[test] fn an_mcp_kind_does_not_match_an_inference_query() { let candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); - assert!(select_admitted(&candidates, CapabilityKind::McpTool, "llama").is_none()); + assert!(front(candidates, CapabilityKind::McpTool, "llama").is_none()); + } + + /// A snapshot of `llama` on east and west, with the given queue depths. + fn two_sites(east: Option, west: Option) -> RouteSnapshot { + let store = grid_signals::LoadStore::new(std::time::Duration::from_secs(60)); + for (site, cluster, load) in [("east", "pool-a", east), ("west", "pool-b", west)] { + if let Some(value) = load { + let line = format!( + r#"{}{{grid_site="{site}",grid_provider="{cluster}"}} {value} 1000"#, + crate::snapshot::LOAD_METRIC + ); + store.ingest_at(&line, 1_000, 1_000, site); + } + } + let mut candidates = one("llama", "east", "pool-a", AdmissionState::NewAndExisting); + candidates.extend(one("llama", "west", "pool-b", AdmissionState::NewAndExisting)); + RouteSnapshot::from_store(candidates, Arc::from("east"), &store, 1_000, 30_000) + } + + fn picks(snapshot: &RouteSnapshot) -> Vec { + (0..2_000) + .map(|turn| { + let chosen = select_spread( + snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |_| true, + &KeepAll, + ) + .unwrap() + .candidate; + chosen.site.to_string() + }) + .collect() + } + + #[test] + fn equal_scores_spread_across_the_tied_candidates() { + for (snapshot, case) in [ + (two_sites(None, None), "unmeasured"), + (two_sites(Some(5.0), Some(5.0)), "measured"), + ] { + let east = picks(&snapshot).iter().filter(|site| *site == "east").count(); + assert!((850..=1_150).contains(&east), "{case}: east took {east} of 2000"); + } + } + + #[test] + fn a_better_score_always_wins() { + assert_eq!(picks(&two_sites(Some(9.0), Some(1.0))), ["west"; 2_000]); + assert_eq!( + picks(&two_sites(Some(1.0), None)), + ["east"; 2_000], + "measured beats unmeasured" + ); + } + + #[test] + fn an_excluded_tie_is_never_picked() { + let mut snapshot = two_sites(None, None); + snapshot.candidates[1].admission_state = AdmissionState::Excluded; + assert_eq!(picks(&snapshot), ["east"; 2_000]); + } + + /// Sites publishing (in flight, capacity), each `None` to publish nothing, as one snapshot. + fn loaded(sites: &[(&str, Option, Option)]) -> RouteSnapshot { + let store = grid_signals::LoadStore::new(std::time::Duration::from_secs(60)); + let mut candidates = Vec::new(); + for (site, in_flight, capacity) in sites { + let cluster = format!("pool-{site}"); + let labels = format!(r#"grid_site="{site}",grid_provider="{cluster}""#); + let saturation = in_flight.zip(*capacity).map(|(held, capacity)| held / capacity); + for (metric, value) in [ + ("grid_provider_saturation_ratio", saturation), + ("grid_provider_capacity_requests", *capacity), + ] { + if let Some(value) = value { + store.ingest_at(&format!("{metric}{{{labels}}} {value} 1000"), 1_000, 1_000, site); + } + } + candidates.extend(one("llama", site, &cluster, AdmissionState::NewAndExisting)); + } + RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000) + } + + /// How many of `draws` requests each site took. + fn shares( + snapshot: &RouteSnapshot, + draws: usize, + narrow: &impl Narrow, + ) -> std::collections::BTreeMap { + let mut taken = std::collections::BTreeMap::new(); + for turn in 0..draws { + let chosen = select_spread( + snapshot, + CapabilityKind::InferenceModel, + "llama", + turn, + |_| true, + narrow, + ) + .unwrap(); + *taken.entry(chosen.candidate.site.to_string()).or_insert(0) += 1; + } + taken + } + + #[test] + fn two_choices_prefer_the_lighter_site_and_never_a_full_one() { + let snapshot = loaded(&[ + ("a", Some(2.0), Some(10.0)), + ("b", Some(8.0), Some(10.0)), + ("c", Some(5.0), Some(10.0)), + ("d", Some(10.0), Some(10.0)), + ]); + let taken = shares(&snapshot, 3_000, &KeepAll); + assert_eq!(taken.get("d"), None, "rho 1 has no room"); + let (a, c) = (taken["a"], taken["c"]); + assert!(a > c && c > 0, "the lighter site takes more: a {a}, c {c}"); + assert_eq!( + taken.get("b"), + None, + "two distinct draws never choose the heaviest of three" + ); + } + + #[test] + fn two_sites_split_by_capacity_over_load_within_a_tenth_and_nine_tenths() { + let even = shares( + &loaded(&[("a", Some(1.0), Some(10.0)), ("b", Some(1.0), Some(10.0))]), + 4_000, + &KeepAll, + ); + assert!( + (1_800..=2_200).contains(&even["a"]), + "equal sites split evenly: {even:?}" + ); + let lopsided = shares( + &loaded(&[("a", Some(0.0), Some(100.0)), ("b", Some(9.0), Some(10.0))]), + 4_000, + &KeepAll, + ); + assert!( + (280..=520).contains(&lopsided["b"]), + "the clamp keeps a tenth for the slow site: {lopsided:?}" + ); + } + + #[test] + fn a_lone_site_with_room_takes_every_request() { + let snapshot = loaded(&[("a", Some(12.0), Some(10.0)), ("b", Some(3.0), Some(10.0))]); + assert_eq!(shares(&snapshot, 200, &KeepAll).get("b"), Some(&200)); + } + + #[test] + fn with_no_room_anywhere_the_overflow_draws_by_capacity() { + let snapshot = loaded(&[("a", Some(30.0), Some(30.0)), ("b", Some(10.0), Some(10.0))]); + let taken = shares(&snapshot, 4_000, &KeepAll); + assert!( + (2_700..=3_300).contains(&taken["a"]), + "a has three quarters of the capacity: {taken:?}" + ); + } + + #[test] + fn sites_without_published_capacity_are_drawn_evenly() { + let snapshot = loaded(&[("a", Some(5.0), None), ("b", Some(1.0), None), ("c", None, None)]); + let taken = shares(&snapshot, 3_000, &KeepAll); + for site in ["a", "b", "c"] { + assert!((850..=1_150).contains(&taken[site]), "no rho, no load input: {taken:?}"); + } + } + + #[test] + fn a_demoted_site_is_never_drawn_while_another_is_healthy() { + let down = std::collections::BTreeSet::from([Arc::from("pool-a")]); + let snapshot = loaded(&[ + ("a", Some(0.0), Some(100.0)), + ("b", Some(5.0), Some(10.0)), + ("c", Some(5.0), Some(10.0)), + ]) + .demote(&down); + assert_eq!(shares(&snapshot, 500, &KeepAll).get("a"), None); + } + + /// Keeps only site `b`, or clears everything when `all` is set. + struct Only { + all: bool, + } + + impl Narrow for Only { + fn keep(&self, sites: &[SiteView<'_>], keep: &mut [bool]) { + for (site, kept) in sites.iter().zip(keep) { + *kept = !self.all + && &*site.candidate.site == "b" + && site.rho < 1.0 + && site.delay.is_none() + && site.latency.ttft_p90.is_none(); + } + } + } + + #[test] + fn a_narrowing_stage_can_only_remove_sites_and_clearing_all_keeps_all() { + let snapshot = loaded(&[ + ("a", Some(1.0), Some(10.0)), + ("b", Some(5.0), Some(10.0)), + ("c", Some(12.0), Some(10.0)), + ]); + assert_eq!(shares(&snapshot, 300, &Only { all: false }).get("b"), Some(&300)); + let taken = shares(&snapshot, 300, &Only { all: true }); + assert_eq!(taken.get("c"), None, "a full site is never added back"); + assert!(taken.contains_key("a") && taken.contains_key("b"), "{taken:?}"); + } + + #[test] + fn a_shed_answers_429_and_an_outage_503_each_with_an_openai_error() { + // Load: an OpenAI client backs off on 429 and retries. + let shed = answered(Refused::Shed); + assert_eq!(shed.0, 429); + assert!(shed.1, "a shed says when to come back"); + assert!( + shed.2.contains(r#""code":"capacity_exhausted""#) && shed.2.contains(r#""type":"rate_limit_exceeded""#), + "{}", + shed.2 + ); + // An outage is not load, so it stays 503. + let outage = answered(Refused::NotReady); + assert_eq!(outage.0, 503); + assert!(outage.1); + assert!( + outage.2.contains(r#""code":"no_healthy_site""#) && outage.2.contains(r#""type":"server_error""#), + "{}", + outage.2 + ); + // An unknown model is the caller's error, with nothing to retry. + let unknown = answered(Refused::UnknownModel); + assert_eq!(unknown.0, 404); + assert!(!unknown.1, "a model that does not exist will not appear"); + assert!(unknown.2.contains(r#""code":"model_not_found""#), "{}", unknown.2); + } + + /// How `reason` is answered: status, whether it says when to retry, and its body. + fn answered(reason: Refused) -> (u16, bool, String) { + match refuse(reason, "llama", 0) { + FilterAction::TerminalResponse(response) => { + let body = String::from_utf8(response.body.clone().unwrap_or_default().to_vec()).unwrap(); + ( + response.status, + response.headers.contains_key(http::header::RETRY_AFTER), + body, + ) + }, + FilterAction::Continue + | FilterAction::Reject(_) + | FilterAction::StreamingTerminalResponse(_) + | FilterAction::Release + | FilterAction::BodyDone => panic!("a refusal is answered here"), + } + } + + #[test] + fn draws_are_uniform_and_seeds_differ() { + let mean = (0..10_000).map(|turn| unit(turn, SALTS.0)).sum::() / 10_000.0; + assert!((0.48..=0.52).contains(&mean), "{mean}"); + assert!((0..10_000).all(|turn| (0.0..1.0).contains(&unit(turn, SALTS.1)))); + assert_ne!(random_seed(), random_seed()); } } diff --git a/gateway/ai-grid-filters/src/serving.rs b/gateway/ai-grid-filters/src/serving.rs index e2aec6c37..bb531ca19 100644 --- a/gateway/ai-grid-filters/src/serving.rs +++ b/gateway/ai-grid-filters/src/serving.rs @@ -23,6 +23,7 @@ use serde::Deserialize; use crate::{ control::{Control, ReloadOutcome, Watcher, watch}, descriptor::CandidateConfig, + health::ClusterHealth, snapshot::RouteSnapshot, }; @@ -110,6 +111,10 @@ pub struct PeerServingConfig { /// Leaf SHA-256 digests the peer must also match, rendered under pin trust only. #[serde(default)] pub pins: Vec, + + /// `host:port` of the peer's gateway, routed to directly; set only while the operator verified it. + #[serde(default)] + pub gateway: Option, } /// The running control plane: the filter's snapshot, the pollers, and the config watch. @@ -122,6 +127,12 @@ pub struct GridRuntime { /// The config file watch, stopped on drop. watcher: Option, + + /// Backend cluster health the route filter publishes and the tick reads. + health: Arc, + + /// Stops the health tick when the runtime drops. + _health_tick: std::sync::mpsc::Sender<()>, } impl GridRuntime { @@ -131,6 +142,12 @@ impl GridRuntime { Arc::clone(&self.snapshot) } + /// The cluster health to register the filter over. + #[must_use] + pub fn health(&self) -> Arc { + Arc::clone(&self.health) + } + /// Apply a new serving config. `None` when it is already applied. /// /// # Errors @@ -180,6 +197,8 @@ pub fn load_serving_config(path: &str) -> Result /// peer's certificate material cannot be read or parsed, or a poller thread /// cannot be spawned. pub fn spawn_grid_routing(config: &GridServingConfig) -> Result { + // The peer scrapers load TLS here, before the server installs the provider. + praxis_tls::provider::install(); let start = Box::new(|peer: &PeerServingConfig, poller: &_, store, refresh| { let scraper = build_scraper(peer)?; spawn_on_thread_held(store, poller, scraper, refresh) @@ -195,15 +214,42 @@ pub(crate) fn start_runtime( ) -> Result { let mut control = Control::new(config, start)?; control.apply(config)?; + let health_tick = tick_health(control.health(), control.refresh(), control.store()) + .map_err(|error| -> FilterError { format!("grid: spawning the health tick: {error}").into() })?; Ok(GridRuntime { snapshot: control.snapshot(), + health: control.health(), control: Arc::new(Mutex::new(control)), watcher: None, + _health_tick: health_tick, }) } -/// Reject peer settings that would silently stop a poller. +/// How often the control step re-reads backend cluster health. +const HEALTH_TICK: Duration = Duration::from_millis(100); + +/// Re-order the snapshot whenever the set of clusters with no healthy endpoint changes. /// +/// Health moves faster than the poll cycle, so it gets its own tick; the returned +/// sender stops the thread when dropped. +fn tick_health( + health: Arc, + refresh: crate::control::Refresh, + store: Arc, +) -> std::io::Result> { + let (stop, stopped) = std::sync::mpsc::channel::<()>(); + std::thread::Builder::new() + .name("grid-health".to_owned()) + .spawn(move || { + while let Err(std::sync::mpsc::RecvTimeoutError::Timeout) = stopped.recv_timeout(HEALTH_TICK) { + if health.update() { + refresh(&store); + } + } + })?; + Ok(stop) +} + /// A zero interval hands `Duration::ZERO` to the interval timer, which panics the /// detached poller thread. A zero timeout fires immediately, so the peer never /// scrapes. Either way that site ages to `+inf` and sorts last, a silent stale @@ -234,6 +280,7 @@ pub(crate) fn validate_peer(peer: &PeerServingConfig) -> Result<(), FilterError> Ok(()) } +/// Grid mTLS to `peer`: this site's identity, verified against the grid CA and the peer's server name. /// Build a peer's mTLS scraper from its config, reading and parsing its /// certificate material. fn build_scraper(peer: &PeerServingConfig) -> Result { @@ -347,6 +394,7 @@ peers: client_cert_path: "/etc/grid/tls.crt".to_owned(), client_key_path: "/etc/grid/tls.key".to_owned(), pins: Vec::new(), + gateway: None, } } @@ -412,4 +460,14 @@ peers: validate_peer(&valid_peer()).expect("a valid peer is accepted"); } + + #[test] + fn a_peer_from_an_operator_without_gateway_support_parses_with_none() { + let peer: PeerServingConfig = serde_yaml::from_str( + "{site: east, addr: '10.0.0.1:9091', server_name: east.grid.internal, authority: east.grid.internal, \ + grid_ca_path: /ca, client_cert_path: /crt, client_key_path: /key}", + ) + .expect("an old operator's peer parses"); + assert!(peer.gateway.is_none(), "no gateway: route through the static cluster"); + } } diff --git a/gateway/ai-grid-filters/src/snapshot.rs b/gateway/ai-grid-filters/src/snapshot.rs index 553f1e980..53395b3ca 100644 --- a/gateway/ai-grid-filters/src/snapshot.rs +++ b/gateway/ai-grid-filters/src/snapshot.rs @@ -6,18 +6,60 @@ //! ([`RouteSnapshot::from_store`]) run off the request path, so the hot path //! snapshots resolved order once rather than reading raw signals per request. -use std::sync::Arc; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; use grid_signals::LoadStore; -use crate::descriptor::RouteCandidate; +use crate::{ + decisions::SiteDecisions, + descriptor::{AdmissionState, CapabilityKind, RouteCandidate}, +}; -/// Load metric that orders candidates. Lower queue depth is a better target. -pub(crate) const LOAD_METRIC: &str = "inference_pool_average_queue_size"; +/// Load metrics that order candidates, current EPP name first. Lower queue depth is a better target. +/// +/// The wire carries each backend's raw metric name, so the consumer maps every name it knows. +pub(crate) const LOAD_METRICS: [&str; 2] = ["llm_d_epp_average_queue_size", "inference_pool_average_queue_size"]; + +/// The load metric current llm-d EPPs export. +#[cfg(test)] +pub(crate) const LOAD_METRIC: &str = LOAD_METRICS[0]; + +/// The requests a provider runs at once, as its operator publishes it. +const CAPACITY_METRIC: &str = "grid_provider_capacity_requests"; + +/// Requests a provider holds over its capacity, as its operator resolves it. +const SATURATION_METRIC: &str = "grid_provider_saturation_ratio"; + +/// Recent latency a provider's operator publishes: TTFT median and 90th percentile, time per +/// output token, and prefill seconds per uncached token. +const LATENCY_METRICS: [&str; 4] = [ + "grid_provider_ttft_p50_seconds", + "grid_provider_ttft_p90_seconds", + "grid_provider_tpot_seconds", + "grid_provider_prefill_seconds_per_token", +]; + +/// A model sheds once every healthy site serving it holds at least this rho. +const SHED_AT: f64 = 1.05; + +/// A shedding model routes again once any healthy site is at or below this rho. +const RESUME_AT: f64 = 0.95; + +/// The operator's per-provider verdict: 0 when the provider cannot serve. +/// +/// Absent reads as ready, so a site whose operator predates it is still routed. +const READY_METRIC: &str = "grid_provider_ready"; /// Lower load is a better routing target. const LOWER_IS_BETTER: bool = true; +/// Score of a candidate whose cluster has no healthy endpoint: after every healthy one, +/// unmeasured included, since `total_cmp` orders a positive NaN above infinity. +const UNHEALTHY: f64 = f64::NAN.abs(); + /// A resolved, pre-ordered candidate list read atomically per request. #[derive(Debug)] pub struct RouteSnapshot { @@ -26,6 +68,15 @@ pub struct RouteSnapshot { /// This gateway's own site identifier. pub local_site: Arc, + + /// Each candidate's score, parallel to `candidates`; equal scores share traffic. + pub scores: Vec, + + /// Each candidate's site decision counters, parallel to `candidates`. + pub decisions: Vec, + + /// Models answering 503 because every healthy site serving them is past full. + pub shedding: BTreeSet>, } impl RouteSnapshot { @@ -34,7 +85,14 @@ impl RouteSnapshot { /// The order is whatever the caller supplies (config order). Used before /// any signals exist and as the cold-start fallback. pub fn from_static(candidates: Vec, local_site: Arc) -> Self { - Self { candidates, local_site } + let scores = vec![f64::INFINITY; candidates.len()]; + Self { + decisions: decisions_for(&candidates), + candidates, + local_site, + scores, + shedding: BTreeSet::new(), + } } /// Order candidates least-loaded-first from the live store, then wrap them. @@ -54,15 +112,162 @@ impl RouteSnapshot { ) -> Self { // Score each candidate once, then sort the pairs: load_of allocates a // store key and scans a window, too costly to repeat inside sort_by. - let mut scored: Vec<(f64, RouteCandidate)> = candidates + let ranked = candidates .into_iter() - .map(|candidate| (Self::load_of(store, &candidate, now_ms, window_ms), candidate)) + .map(|mut candidate| { + if Self::reported_unready(store, &candidate) { + candidate.admission_state = AdmissionState::Excluded; + } + candidate.capacity = Self::capacity_of(store, &candidate); + candidate.rho = Self::fresh(store, &candidate, SATURATION_METRIC, now_ms, window_ms); + let [ttft_p50, ttft_p90, tpot, prefill] = + LATENCY_METRICS.map(|metric| Self::fresh(store, &candidate, metric, now_ms, window_ms)); + candidate.latency = crate::descriptor::Latency { + ttft_p50, + ttft_p90, + tpot, + prefill_per_token: prefill, + }; + let load = if candidate.admission_state == AdmissionState::NewAndExisting { + Self::load_of(store, &candidate, now_ms, window_ms) + } else { + UNHEALTHY + }; + (load, candidate) + }) .collect(); - scored.sort_by(|(left, _), (right, _)| left.total_cmp(right)); - let ordered = scored.into_iter().map(|(_, candidate)| candidate).collect(); + Self::ranked(ranked, local_site) + } + + /// The candidate's published capacity, `None` when none is published or it is not a positive count. + fn capacity_of(store: &LoadStore, candidate: &RouteCandidate) -> Option { + let key = LoadStore::key(&candidate.site, &candidate.cluster); + store + .latest(&key, CAPACITY_METRIC) + .map(|sample| sample.value) + .filter(|capacity| capacity.is_finite() && *capacity >= 1.0) + } + + /// The candidate's latest `metric` sample, `None` when absent or older than `window_ms`. + fn fresh(store: &LoadStore, candidate: &RouteCandidate, metric: &str, now_ms: i64, window_ms: i64) -> Option { + let key = LoadStore::key(&candidate.site, &candidate.cluster); + store + .latest(&key, metric) + .filter(|sample| now_ms.saturating_sub(sample.at_ms) <= window_ms && sample.value.is_finite()) + .map(|sample| sample.value.max(0.0)) + } + + /// This snapshot with the models to shed, given the set `previous` shed. + /// + /// A model sheds when every admitted, healthy site serving it has a fresh rho of at + /// least 1.05, and routes again once one reaches 0.95. Between the two it keeps its + /// previous state, so a value near the threshold does not flap. A model with any + /// healthy site of unknown rho never sheds: that site takes the overflow. + #[must_use] + pub(crate) fn shed(mut self, previous: &BTreeSet>) -> Self { + // Per model: whether every healthy site is measured, and their rho. + let mut models: BTreeMap<&Arc, (bool, Vec)> = BTreeMap::new(); + let healthy = self.candidates.iter().zip(&self.scores).filter(|(candidate, score)| { + candidate.kind == CapabilityKind::InferenceModel + && candidate.admission_state == AdmissionState::NewAndExisting + && !score.is_nan() + }); + for (candidate, _) in healthy { + let (measured, loads) = models.entry(&candidate.name).or_insert((true, Vec::new())); + match candidate.rho { + Some(rho) => loads.push(rho), + None => *measured = false, + } + } + self.shedding = models + .into_iter() + .filter(|(model, (measured, loads))| { + let full = loads.iter().all(|rho| *rho >= SHED_AT); + let room = loads.iter().any(|rho| *rho <= RESUME_AT); + *measured && !loads.is_empty() && (full || (previous.contains(*model) && !room)) + }) + .map(|(model, _)| Arc::clone(model)) + .collect(); + self + } + + /// Whether the candidate's latest readiness sample says it cannot serve. + /// + /// The latest at any age, not the worst in a window: a recovered provider rejoins + /// on its next poll, and a partitioned peer keeps its last 0 rather than reading as + /// ready once it ages out. A 0 stands until something newer arrives for the + /// provider: a later reading without the series means its peer stopped publishing + /// readiness, which reads as ready, like a peer that never published it. + fn reported_unready(store: &LoadStore, candidate: &RouteCandidate) -> bool { + let key = LoadStore::key(&candidate.site, &candidate.cluster); + store + .latest(&key, READY_METRIC) + .is_some_and(|ready| ready.value < 1.0 && store.newest_at(&key).is_none_or(|newest| newest <= ready.at_ms)) + } + + /// This snapshot with every candidate on a `down` cluster ordered after all the rest. + /// + /// Last rather than dropped: while any candidate is healthy it is never chosen, and + /// when none is, the request still has somewhere to go. + #[must_use] + pub(crate) fn demote(self, down: &BTreeSet>) -> Self { + self.demote_where(down, |candidate| Some(&candidate.cluster)) + } + + /// This snapshot with every candidate whose `key` is in `down` ordered last. + fn demote_where(self, down: &BTreeSet>, key: impl Fn(&RouteCandidate) -> Option<&Arc>) -> Self { + if down.is_empty() { + return self; + } + let ranked = self + .scores + .into_iter() + .zip(self.candidates) + .map(|(load, candidate)| { + let demoted = if key(&candidate).is_some_and(|key| down.contains(key)) { + UNHEALTHY + } else { + load + }; + (demoted, candidate) + }) + .collect(); + Self::ranked(ranked, self.local_site) + } + + /// This snapshot, after setting `grid_route_site_score` for each of its site/cluster pairs. + /// + /// The gauge is the score the order used, NaN when the candidate is excluded or + /// demoted. A pair in `published` but no longer in the topology is set to NaN, + /// since the exporter keeps a series until restart. `published` becomes this + /// snapshot's pairs. + #[must_use] + pub(crate) fn published(self, published: &mut Gauged) -> Self { + let mut current = BTreeSet::new(); + for (candidate, score) in self.candidates.iter().zip(&self.scores) { + let key = (Arc::clone(&candidate.site), Arc::clone(&candidate.cluster)); + // The front entry for a pair is its best; later ones are other models. + if current.insert(key.clone()) { + site_score(&key, *score); + } + } + for gone in published.difference(¤t) { + site_score(gone, f64::NAN); + } + *published = current; + self + } + + /// Sort `ranked` ascending by score, stable among equals, and wrap it. + fn ranked(mut ranked: Vec<(f64, RouteCandidate)>, local_site: Arc) -> Self { + ranked.sort_by(|(left, _), (right, _)| left.total_cmp(right)); + let (scores, ordered): (Vec, Vec) = ranked.into_iter().unzip(); Self { + decisions: decisions_for(&ordered), candidates: ordered, local_site, + scores, + shedding: BTreeSet::new(), } } @@ -71,12 +276,32 @@ impl RouteSnapshot { /// `+inf` makes an unmeasured candidate sort last under an ascending order. fn load_of(store: &LoadStore, candidate: &RouteCandidate, now_ms: i64, window_ms: i64) -> f64 { let key = LoadStore::key(&candidate.site, &candidate.cluster); - store - .window_worst(&key, LOAD_METRIC, now_ms, window_ms, LOWER_IS_BETTER) + LOAD_METRICS + .iter() + .find_map(|metric| store.window_worst(&key, metric, now_ms, window_ms, LOWER_IS_BETTER)) .unwrap_or(f64::INFINITY) } } +/// Decision counters for each of `candidates`, in order. +fn decisions_for(candidates: &[RouteCandidate]) -> Vec { + candidates + .iter() + .map(|candidate| SiteDecisions::new(&candidate.site)) + .collect() +} + +/// A candidate's site and cluster, the labels of its score gauge. +type SitePair = (Arc, Arc); + +/// The site/cluster pairs with a published score gauge. +pub(crate) type Gauged = BTreeSet; + +/// Set `grid_route_site_score` for one site/cluster pair. +fn site_score((site, cluster): &SitePair, score: f64) { + metrics::gauge!("grid_route_site_score", "site" => Arc::clone(site), "cluster" => Arc::clone(cluster)).set(score); +} + #[cfg(test)] #[expect(clippy::allow_attributes, reason = "blanket test suppressions")] #[allow( @@ -85,13 +310,14 @@ impl RouteSnapshot { clippy::indexing_slicing, clippy::panic, clippy::min_ident_chars, + clippy::too_many_lines, reason = "tests" )] mod tests { use std::time::Duration; use super::*; - use crate::descriptor::{CandidateConfig, CapabilityKind, validate_candidates}; + use crate::descriptor::{CandidateConfig, validate_candidates}; /// One QUEUE sample for `site`/`cluster` at `at_ms`. fn line(site: &str, cluster: &str, value: f64, at_ms: i64) -> String { @@ -101,6 +327,7 @@ mod tests { /// A validated candidate for model `name` served by `site`/`cluster`. fn cand(name: &str, site: &str, cluster: &str) -> CandidateConfig { CandidateConfig { + admission: AdmissionState::default(), cluster: cluster.to_owned(), credential: None, fresh: true, @@ -110,6 +337,188 @@ mod tests { } } + #[test] + fn a_cluster_with_no_healthy_endpoint_is_never_chosen_while_another_is_healthy() { + let store = LoadStore::new(Duration::from_secs(60)); + // site-a is the least loaded but down; site-d is unmeasured. + store.ingest_at(&line("local", "site-a", 0.0, 1_000), 1_000, 1_000, "local"); + store.ingest_at(&line("local", "site-b", 50.0, 1_000), 1_000, 1_000, "local"); + let candidates = validate_candidates(vec![ + cand("m", "local", "site-a"), + cand("m", "local", "site-b"), + cand("m", "site-d", "site-d"), + ]) + .unwrap(); + let down = BTreeSet::from([Arc::from("site-a")]); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("local"), &store, 1_000, 60_000).demote(&down); + let order: Vec<&str> = snapshot.candidates.iter().map(|c| &*c.cluster).collect(); + assert_eq!(order, ["site-b", "site-d", "site-a"], "down sorts after unmeasured"); + for turn in 0..10 { + let picked = crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + turn, + |_| true, + &crate::route::KeepAll, + ) + .unwrap() + .candidate; + assert_ne!(&*picked.cluster, "site-a", "turn {turn}"); + } + } + + /// One readiness sample for `site`/`cluster` at `at_ms`. + fn ready(site: &str, cluster: &str, value: f64, at_ms: i64) -> String { + format!(r#"{READY_METRIC}{{grid_site="{site}",grid_provider="{cluster}"}} {value} {at_ms}"#) + } + + #[test] + fn a_provider_reported_unready_is_excluded_until_a_newer_sample_says_ready() { + let store = LoadStore::new(Duration::from_secs(60)); + // site-b is the least loaded, but its operator says it cannot serve. + store.ingest_at(&line("site-a", "pool-a", 5.0, 1_000), 1_000, 1_000, "site-a"); + store.ingest_at(&line("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + store.ingest_at(&ready("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + let candidates = || { + validate_candidates(vec![ + cand("m", "site-a", "pool-a"), + cand("m", "site-b", "pool-b"), + cand("m", "site-d", "pool-d"), + ]) + .unwrap() + }; + let snapshot = RouteSnapshot::from_store(candidates(), Arc::from("hub"), &store, 1_000, 30_000); + let order: Vec<&str> = snapshot.candidates.iter().map(|c| &*c.cluster).collect(); + assert_eq!(order, ["pool-a", "pool-d", "pool-b"], "unready sorts last"); + for turn in 0..10 { + let picked = crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + turn, + |_| true, + &crate::route::KeepAll, + ) + .unwrap() + .candidate; + assert_ne!(&*picked.cluster, "pool-b", "turn {turn}"); + } + + store.ingest_at(&ready("site-b", "pool-b", 1.0, 2_000), 2_000, 2_000, "site-b"); + let recovered = RouteSnapshot::from_store(candidates(), Arc::from("hub"), &store, 2_000, 30_000); + assert_eq!( + &*recovered.candidates[0].cluster, "pool-b", + "back on the next sample, not after the window" + ); + } + + #[test] + fn a_not_ready_verdict_holds_through_silence_and_clears_when_the_peer_stops_publishing_it() { + let store = LoadStore::new(Duration::from_secs(60)); + store.ingest_at(&line("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + store.ingest_at(&ready("site-b", "pool-b", 0.0, 1_000), 1_000, 1_000, "site-b"); + let order = |now| { + let candidates = + validate_candidates(vec![cand("m", "site-b", "pool-b"), cand("m", "site-a", "pool-a")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, now, 30_000); + snapshot.candidates.first().map(|c| c.cluster.to_string()) + }; + // Partitioned: nothing newer arrives, long after the load window. + assert_eq!( + order(120_000).as_deref(), + Some("pool-a"), + "the last 0 stands through silence" + ); + // Healed, but the peer no longer publishes readiness. + store.ingest_at(&line("site-b", "pool-b", 0.0, 121_000), 121_000, 121_000, "site-b"); + assert_eq!( + order(121_000).as_deref(), + Some("pool-b"), + "a newer reading without the series is ready" + ); + } + + #[test] + fn capacity_comes_from_the_published_series_and_is_unknown_otherwise() { + let store = LoadStore::new(Duration::from_secs(60)); + let capacity = |site: &str, cluster: &str, value: f64| { + format!(r#"grid_provider_capacity_requests{{grid_site="{site}",grid_provider="{cluster}"}} {value} 1000"#) + }; + store.ingest_at(&capacity("site-a", "pool-a", 64.0), 1_000, 1_000, "site-a"); + store.ingest_at(&capacity("site-b", "pool-b", 0.0), 1_000, 1_000, "site-b"); + let candidates = validate_candidates(vec![ + cand("m", "site-a", "pool-a"), + cand("m", "site-b", "pool-b"), + cand("m", "site-d", "pool-d"), + ]) + .unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000); + let by_site: Vec<(&str, Option)> = snapshot.candidates.iter().map(|c| (&*c.site, c.capacity)).collect(); + assert!(by_site.contains(&("site-a", Some(64.0)))); + assert!(by_site.contains(&("site-b", None)), "0 is not a capacity"); + assert!(by_site.contains(&("site-d", None)), "none published"); + } + + #[test] + fn an_excluded_candidate_from_the_serving_config_is_never_chosen() { + let store = LoadStore::new(Duration::from_secs(60)); + let mut excluded = cand("m", "site-b", "pool-b"); + excluded.admission = AdmissionState::Excluded; + let candidates = validate_candidates(vec![excluded, cand("m", "site-a", "pool-a")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000); + for turn in 0..4 { + let picked = crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + turn, + |_| true, + &crate::route::KeepAll, + ) + .unwrap() + .candidate; + assert_eq!(&*picked.cluster, "pool-a"); + } + } + + #[test] + fn when_every_cluster_is_down_one_is_still_chosen() { + let store = LoadStore::new(Duration::from_secs(60)); + let candidates = validate_candidates(vec![cand("m", "local", "site-a"), cand("m", "local", "site-b")]).unwrap(); + let down = BTreeSet::from([Arc::from("site-a"), Arc::from("site-b")]); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("local"), &store, 1_000, 60_000).demote(&down); + assert!( + crate::route::select_spread( + &snapshot, + CapabilityKind::InferenceModel, + "m", + 0, + |_| true, + &crate::route::KeepAll + ) + .is_some() + ); + } + + #[test] + fn a_site_reporting_the_legacy_load_name_is_still_ordered() { + let store = LoadStore::new(Duration::from_secs(60)); + let legacy = |site: &str, cluster: &str, value: f64| { + format!(r#"inference_pool_average_queue_size{{grid_site="{site}",grid_provider="{cluster}"}} {value} 1000"#) + }; + store.ingest_at(&legacy("east", "pool-a", 90.0), 1_000, 1_000, "east"); + store.ingest_at(&line("west", "pool-b", 10.0, 1_000), 1_000, 1_000, "west"); + let candidates = validate_candidates(vec![cand("m", "east", "pool-a"), cand("m", "west", "pool-b")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("local"), &store, 1_000, 60_000); + let order: Vec<&str> = snapshot.candidates.iter().map(|c| &*c.site).collect(); + assert_eq!( + order, + ["west", "east"], + "both names measure load; neither sorts as unmeasured" + ); + } + #[test] fn the_least_loaded_site_sorts_first() { let store = LoadStore::new(Duration::from_secs(60)); @@ -159,4 +568,79 @@ mod tests { ); assert_eq!(&*snap.candidates[1].site, "west"); } + + /// One snapshot of sites `a` and `b` at the given rho, with capacity 100. + fn at_rho(rhos: [Option; 2]) -> RouteSnapshot { + let store = LoadStore::new(Duration::from_secs(60)); + for (site, rho) in ["a", "b"].iter().zip(rhos) { + let labels = format!(r#"grid_site="{site}",grid_provider="pool-{site}""#); + store.ingest_at( + &format!("grid_provider_capacity_requests{{{labels}}} 100 1000"), + 1_000, + 1_000, + site, + ); + if let Some(rho) = rho { + store.ingest_at( + &format!("grid_provider_saturation_ratio{{{labels}}} {rho} 1000"), + 1_000, + 1_000, + site, + ); + } + } + let candidates = validate_candidates(vec![cand("m", "a", "pool-a"), cand("m", "b", "pool-b")]).unwrap(); + RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000) + } + + #[test] + fn a_model_sheds_past_full_and_routes_again_once_a_site_has_room() { + let none = BTreeSet::new(); + let shed = BTreeSet::from([Arc::from("m")]); + assert!( + at_rho([Some(1.10), Some(1.06)]).shed(&none).shedding.contains("m"), + "both past 1.05" + ); + assert!( + at_rho([Some(1.10), Some(1.0)]).shed(&none).shedding.is_empty(), + "one below 1.05 does not start a shed" + ); + assert!( + at_rho([Some(1.10), Some(1.0)]).shed(&shed).shedding.contains("m"), + "nor does it end one" + ); + assert!( + at_rho([Some(1.10), Some(0.95)]).shed(&shed).shedding.is_empty(), + "0.95 ends it" + ); + assert!( + at_rho([Some(1.10), None]).shed(&shed).shedding.is_empty(), + "an unmeasured site never sheds" + ); + } + + #[test] + fn published_latency_reaches_the_candidate_and_absent_stays_unknown() { + let store = LoadStore::new(Duration::from_secs(60)); + let labels = r#"grid_site="a",grid_provider="pool-a""#; + store.ingest_at( + &format!("grid_provider_ttft_p90_seconds{{{labels}}} 0.8 1000"), + 1_000, + 1_000, + "a", + ); + store.ingest_at( + &format!("grid_provider_tpot_seconds{{{labels}}} 0.02 1000"), + 1_000, + 1_000, + "a", + ); + let candidates = validate_candidates(vec![cand("m", "a", "pool-a")]).unwrap(); + let snapshot = RouteSnapshot::from_store(candidates, Arc::from("hub"), &store, 1_000, 30_000); + let latency = snapshot.candidates[0].latency; + assert_eq!(latency.ttft_p90, Some(0.8)); + assert_eq!(latency.tpot, Some(0.02)); + assert_eq!(latency.ttft_p50, None, "unpublished"); + assert_eq!(latency.prefill_per_token, None, "unpublished"); + } } diff --git a/gateway/ai-grid-filters/testdata/serving-config.json b/gateway/ai-grid-filters/testdata/serving-config.json index 570b01ab2..40ae0a517 100644 --- a/gateway/ai-grid-filters/testdata/serving-config.json +++ b/gateway/ai-grid-filters/testdata/serving-config.json @@ -1,7 +1,7 @@ { "local_site": "site-a", "window_secs": 60, - "load_window_ms": 30000, + "load_window_ms": 17000, "candidates": [ { "kind": "inference_model", diff --git a/gateway/src/main.rs b/gateway/src/main.rs index 629a2da2a..8f90299e0 100644 --- a/gateway/src/main.rs +++ b/gateway/src/main.rs @@ -17,6 +17,8 @@ use std::process::ExitCode; use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; use tracing::info; +mod metrics_listener; + /// Log line emitted once tracing is up; the startup test waits for it. const STARTUP_MESSAGE: &str = "starting grid-gateway"; @@ -42,6 +44,11 @@ fn main() -> ExitCode { let log_output = config.runtime.logging.output; info!(version = env!("CARGO_PKG_VERSION"), "{STARTUP_MESSAGE}"); + // Before grid routing starts, so its metrics record into the installed recorder. + if let Err(err) = start_metrics_listener(&config) { + return praxis::report_fatal(&err, log_output); + } + let mut registry = praxis_filter::FilterRegistry::with_builtins(); praxis_ai_filters::register_ai_filters(&mut registry, None); @@ -65,6 +72,27 @@ fn main() -> ExitCode { result.map_or_else(|err| praxis::report_fatal(&err, log_output), |()| ExitCode::SUCCESS) } +/// Start the opt-in metrics listener when its env vars are set. +/// +/// # Errors +/// +/// Returns the settings, port, cert, or bind error. +fn start_metrics_listener(config: &Config) -> Result<(), String> { + let listener = metrics_listener::MetricsListener::from_env(|name| std::env::var(name).ok())?; + // Praxis installs the recorder only when the admin server starts, after grid routing + // publishes its first snapshot, so install it now when anything will serve metrics. + if listener.is_some() || config.admin.address.is_some() { + praxis_protocol::http::pingora::metrics::install_prometheus_recorder(); + } + let Some(listener) = listener else { + return Ok(()); + }; + listener.check_ports(config)?; + // The thread serves for the life of the process. + drop(listener.spawn()?); + Ok(()) +} + /// Start the cross-site pollers and register `grid_site_route` over their snapshot. /// /// # Errors @@ -77,7 +105,7 @@ fn start_grid_routing( ) -> Result { let config = ai_grid_filters::load_serving_config(path)?; let mut runtime = ai_grid_filters::spawn_grid_routing(&config)?; - ai_grid_filters::register_grid_filters(registry, runtime.snapshot())?; + ai_grid_filters::register_grid_filters(registry, runtime.snapshot(), runtime.health())?; // The operator rewrites the file on membership and topology changes. runtime .watch(path, SERVING_RELOAD_INTERVAL) diff --git a/gateway/src/metrics_listener.rs b/gateway/src/metrics_listener.rs new file mode 100644 index 000000000..49a1c030a --- /dev/null +++ b/gateway/src/metrics_listener.rs @@ -0,0 +1,471 @@ +//! Opt-in TLS listener that serves only `GET /metrics`. +//! +//! The Praxis admin listener also serves kv, pipelines, and log-level with no +//! auth, and refuses a non-loopback `Host`, so an in-cluster scraper cannot use +//! it. This listener renders the same Prometheus registry on its own port. + +use std::{ + borrow::Cow, + io, + net::{SocketAddr, TcpListener as StdTcpListener}, + sync::Arc, + thread::JoinHandle, + time::Duration, +}; + +use praxis_core::config::Config; +use praxis_tls::{CertKeyPair, reload::ReloadableCertResolver, watcher::CertWatcher}; +use tokio::{ + io::{AsyncRead, AsyncReadExt as _, AsyncWriteExt as _}, + net::{TcpListener, TcpStream}, + sync::Semaphore, + time::timeout, +}; +use tokio_rustls::{ + TlsAcceptor, + rustls::{ServerConfig, crypto::CryptoProvider}, +}; +use tracing::{debug, info, warn}; + +/// Env var naming the listener's bind address. Unset leaves the listener off. +const ADDR_ENV: &str = "GRID_METRICS_ADDR"; + +/// Env var naming the serving certificate (PEM chain). +const CERT_ENV: &str = "GRID_METRICS_TLS_CERT"; + +/// Env var naming the serving key (PEM). +const KEY_ENV: &str = "GRID_METRICS_TLS_KEY"; + +/// Concurrent connections. A scraper needs one, and extras are closed on accept. +const MAX_CONNECTIONS: usize = 4; + +/// Deadline for the TLS handshake. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); + +/// Deadline for a whole connection, handshake to close, so no client holds a slot longer. +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(5); + +/// Largest request head read. A longer one gets 431. +const MAX_HEAD: usize = 8 * 1024; + +/// Most request headers parsed. +const MAX_HEADERS: usize = 32; + +/// Pause after a failed accept, so a persistent error such as EMFILE does not spin. +const ACCEPT_BACKOFF: Duration = Duration::from_millis(100); + +/// The listener's settings, read from the environment. +#[derive(Debug)] +pub(crate) struct MetricsListener { + /// Bind address. + addr: SocketAddr, + + /// Serving certificate and key, reloaded on change. + pair: CertKeyPair, +} + +impl MetricsListener { + /// The listener settings from `lookup`, or `None` when the address is unset. + /// + /// # Errors + /// + /// Returns a message for an unparsable address or a missing cert or key. + pub(crate) fn from_env Option>(lookup: F) -> Result, String> { + let Some(addr) = lookup(ADDR_ENV).filter(|addr| !addr.is_empty()) else { + return Ok(None); + }; + let addr = addr + .parse() + .map_err(|err| format!("{ADDR_ENV} {addr:?} is not an ip:port: {err}"))?; + let path = |name: &str| { + lookup(name) + .filter(|path| !path.is_empty()) + .ok_or_else(|| format!("{ADDR_ENV} needs {name}: the metrics listener serves TLS only")) + }; + Ok(Some(Self { + addr, + pair: CertKeyPair { + cert_path: path(CERT_ENV)?, + default: false, + key_path: path(KEY_ENV)?, + server_names: Vec::new(), + }, + })) + } + + /// Refuse a port that a Praxis listener or the admin listener already binds. + /// + /// # Errors + /// + /// Returns a message naming the colliding address, or one with no readable port. + pub(crate) fn check_ports(&self, config: &Config) -> Result<(), String> { + let taken = config + .listeners + .iter() + .map(|listener| listener.address.as_str()) + .chain(config.admin.address.as_deref()); + for address in taken { + let port = address + .rsplit_once(':') + .and_then(|(_, port)| port.parse::().ok()) + .ok_or_else(|| format!("Praxis address {address:?} has no port to check {ADDR_ENV} against"))?; + if port == self.addr.port() { + return Err(format!( + "{ADDR_ENV} {} collides with Praxis address {address}", + self.addr + )); + } + } + Ok(()) + } + + /// Bind the listener and serve it on its own thread. + /// + /// # Errors + /// + /// Returns a message when the cert does not load, no crypto provider is + /// installed, or the address does not bind. + pub(crate) fn spawn(self) -> Result, String> { + let resolver = ReloadableCertResolver::new(&self.pair) + .map_err(|err| format!("metrics listener certificate {}: {err}", self.pair.cert_path))?; + let current = resolver.arc(); + let acceptor = TlsAcceptor::from(Arc::new(server_config(resolver)?)); + let listener = StdTcpListener::bind(self.addr) + .and_then(|listener| listener.set_nonblocking(true).map(|()| listener)) + .map_err(|err| format!("metrics listener bind {}: {err}", self.addr))?; + // A dropped sender leaves the watcher running for the life of the process. + let (_, shutdown) = tokio::sync::watch::channel(false); + drop(CertWatcher::spawn(current, self.pair, None, shutdown)); + let addr = self.addr; + std::thread::Builder::new() + .name("grid-metrics".to_owned()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime, + Err(err) => { + warn!(error = %err, "metrics listener runtime failed to start"); + return; + }, + }; + info!(%addr, "metrics listener serving"); + runtime.block_on(serve(listener, acceptor)); + }) + .map_err(|err| format!("metrics listener thread: {err}")) + } +} + +/// The listener's TLS config, over the process crypto provider so FIPS mode carries over. +/// +/// # Errors +/// +/// Returns a message when no provider is installed, it rejects the protocol versions, +/// or FIPS is required and the config is not FIPS. +fn server_config(resolver: ReloadableCertResolver) -> Result { + let provider = CryptoProvider::get_default() + .ok_or_else(|| "metrics listener: no rustls crypto provider installed".to_owned())?; + let mut config = ServerConfig::builder_with_provider(Arc::clone(provider)) + .with_safe_default_protocol_versions() + .map_err(|err| format!("metrics listener TLS: {err}"))? + .with_no_client_auth() + .with_cert_resolver(Arc::new(resolver)); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + // As Praxis sets on its own listeners: SP 800-52r2 needs EMS for TLS 1.2. + config.require_ems = true; + if praxis_tls::provider::required() && !config.fips() { + return Err("metrics listener: FIPS is required but its TLS config is not FIPS".to_owned()); + } + Ok(config) +} + +/// Register the bound socket with the runtime and serve it. +async fn serve(listener: StdTcpListener, acceptor: TlsAcceptor) { + match TcpListener::from_std(listener) { + Ok(listener) => accept_loop(&listener, &acceptor).await, + Err(err) => warn!(error = %err, "metrics listener failed to register its socket"), + } +} + +/// Accept connections forever, closing any beyond [`MAX_CONNECTIONS`]. +#[expect(clippy::infinite_loop, reason = "serves for the life of the process")] +async fn accept_loop(listener: &TcpListener, acceptor: &TlsAcceptor) { + let slots = Arc::new(Semaphore::new(MAX_CONNECTIONS)); + loop { + let (stream, peer) = match listener.accept().await { + Ok(accepted) => accepted, + Err(err) => { + warn!(error = %err, "metrics listener accept failed"); + tokio::time::sleep(ACCEPT_BACKOFF).await; + continue; + }, + }; + let Ok(permit) = Arc::clone(&slots).try_acquire_owned() else { + debug!(%peer, "metrics listener at capacity, closing connection"); + drop(stream); + continue; + }; + let acceptor = acceptor.clone(); + drop(tokio::spawn(async move { + if let Err(err) = Box::pin(handle(stream, &acceptor)).await { + debug!(%peer, error = %err, "metrics connection failed"); + } + drop(permit); + })); + } +} + +/// Serve one request on `stream` and close it. +/// +/// # Errors +/// +/// Returns the I/O error or timeout that ended the connection. +async fn handle(stream: TcpStream, acceptor: &TlsAcceptor) -> io::Result<()> { + // Boxed: the TLS stream makes this future several KiB. + within( + CONNECTION_TIMEOUT, + Box::pin(async { + let mut tls = within(HANDSHAKE_TIMEOUT, acceptor.accept(stream)).await?; + let (head, body) = read_request(&mut tls).await?.into_parts(); + tls.write_all(head.as_bytes()).await?; + tls.write_all(body.as_bytes()).await?; + tls.shutdown().await + }), + ) + .await +} + +/// `future`'s result, or a `TimedOut` error after `limit`. +/// +/// # Errors +/// +/// Returns `future`'s error, or `TimedOut`. +async fn within>>(limit: Duration, future: F) -> io::Result { + timeout(limit, future) + .await + .unwrap_or_else(|_elapsed| Err(io::ErrorKind::TimedOut.into())) +} + +/// Read a request head of at most [`MAX_HEAD`] bytes and decide the reply. +/// +/// # Errors +/// +/// Returns the read error, or `UnexpectedEof` when the peer closes mid-head. +async fn read_request(stream: &mut S) -> io::Result { + let mut head = vec![0_u8; MAX_HEAD]; + let mut filled = 0_usize; + loop { + let Some(spare) = head.get_mut(filled..).filter(|spare| !spare.is_empty()) else { + return Ok(Reply::HeadTooLarge); + }; + let read = stream.read(spare).await?; + if read == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + filled = filled.checked_add(read).unwrap_or(MAX_HEAD); + let mut headers = [httparse::EMPTY_HEADER; MAX_HEADERS]; + let mut request = httparse::Request::new(&mut headers); + match request.parse(head.get(..filled).unwrap_or_default()) { + Ok(httparse::Status::Complete(_)) => return Ok(Reply::route(request.method, request.path)), + Ok(httparse::Status::Partial) => {}, + Err(httparse::Error::TooManyHeaders) => return Ok(Reply::HeadTooLarge), + Err(_) => return Ok(Reply::BadRequest), + } + } +} + +/// The listener's possible answers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Reply { + /// 200 with the Prometheus exposition, or 503 before the recorder exists. + Metrics, + /// 404 for any path but `/metrics`. + NotFound, + /// 405 for `/metrics` with a method other than GET. + MethodNotAllowed, + /// 400 for an unparsable head. + BadRequest, + /// 431 for a head over [`MAX_HEAD`] or [`MAX_HEADERS`]. + HeadTooLarge, +} + +impl Reply { + /// The reply for `method` and `path`, ignoring a query string on `/metrics`. + fn route(method: Option<&str>, path: Option<&str>) -> Self { + let path = path.map(|path| path.split_once('?').map_or(path, |(path, _)| path)); + match (method, path) { + (Some("GET"), Some("/metrics")) => Self::Metrics, + (_, Some("/metrics")) => Self::MethodNotAllowed, + _ => Self::NotFound, + } + } + + /// The HTTP/1.1 response head and body, closing the connection. + fn into_parts(self) -> (String, Cow<'static, str>) { + let (status, body) = match self { + Self::Metrics => { + return match praxis_protocol::http::pingora::metrics::render_prometheus() { + Some(body) => response("200 OK", "", "text/plain; version=0.0.4; charset=utf-8", body.into()), + None => response( + "503 Service Unavailable", + "", + "text/plain", + "metrics recorder not installed\n".into(), + ), + }; + }, + Self::NotFound => ("404 Not Found", "not found\n"), + Self::MethodNotAllowed => ("405 Method Not Allowed", "method not allowed\n"), + Self::BadRequest => ("400 Bad Request", "bad request\n"), + Self::HeadTooLarge => ("431 Request Header Fields Too Large", "request head too large\n"), + }; + let allow = if self == Self::MethodNotAllowed { + "Allow: GET\r\n" + } else { + "" + }; + response(status, allow, "text/plain", body.into()) + } +} + +/// An HTTP/1.1 response head with `extra` header lines, closing the connection, and its body. +fn response(status: &str, extra: &str, content_type: &str, body: Cow<'static, str>) -> (String, Cow<'static, str>) { + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\nContent-Length: {}\r\n{extra}Connection: close\r\n\r\n", + body.len() + ); + (head, body) +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::unwrap_used, clippy::expect_used, reason = "tests")] +mod tests { + use std::path::PathBuf; + + use super::*; + + fn env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let pairs: Vec<(String, String)> = pairs + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect(); + move |name| { + pairs + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + } + } + + #[test] + fn an_unset_address_leaves_the_listener_off() { + assert!(MetricsListener::from_env(env(&[])).unwrap().is_none(), "unset"); + assert!( + MetricsListener::from_env(env(&[(ADDR_ENV, "")])).unwrap().is_none(), + "empty" + ); + } + + #[test] + fn an_address_without_a_cert_or_key_is_refused() { + let no_key = MetricsListener::from_env(env(&[(ADDR_ENV, "0.0.0.0:9443"), (CERT_ENV, "/c")])).unwrap_err(); + assert!(no_key.contains(KEY_ENV), "{no_key}"); + let no_cert = MetricsListener::from_env(env(&[(ADDR_ENV, "0.0.0.0:9443"), (KEY_ENV, "/k")])).unwrap_err(); + assert!(no_cert.contains(CERT_ENV), "{no_cert}"); + let hostname = MetricsListener::from_env(env(&[(ADDR_ENV, "metrics:9443")])).unwrap_err(); + assert!(hostname.contains("ip:port"), "{hostname}"); + } + + #[test] + fn a_port_praxis_binds_is_refused() { + let listener = MetricsListener::from_env(env(&[(ADDR_ENV, "0.0.0.0:9443"), (CERT_ENV, "/c"), (KEY_ENV, "/k")])) + .unwrap() + .unwrap(); + let config = |yaml: &str| Config::from_yaml(yaml).expect("config"); + let base = "listeners:\n - name: web\n address: \"0.0.0.0:8080\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters: []\n"; + listener.check_ports(&config(base)).unwrap(); + let clash = base.replace("8080", "9443"); + assert!( + listener.check_ports(&config(&clash)).unwrap_err().contains("collides"), + "listener" + ); + let admin = format!("{base}admin:\n address: \"127.0.0.1:9443\"\n"); + assert!( + listener.check_ports(&config(&admin)).unwrap_err().contains("collides"), + "admin" + ); + } + + async fn reply_to(request: &[u8]) -> Reply { + let mut reader = request; + read_request(&mut reader).await.unwrap() + } + + #[tokio::test] + async fn only_get_metrics_is_served() { + assert_eq!( + reply_to(b"GET /metrics HTTP/1.1\r\nHost: x\r\n\r\n").await, + Reply::Metrics + ); + assert_eq!(reply_to(b"GET /metrics?x=1 HTTP/1.1\r\n\r\n").await, Reply::Metrics); + assert_eq!( + reply_to(b"POST /metrics HTTP/1.1\r\n\r\n").await, + Reply::MethodNotAllowed + ); + assert_eq!( + reply_to(b"HEAD /metrics HTTP/1.1\r\n\r\n").await, + Reply::MethodNotAllowed + ); + for path in ["/", "/api/kv", "/ready", "/metrics/", "/metricsx"] { + let request = format!("GET {path} HTTP/1.1\r\n\r\n"); + assert_eq!(reply_to(request.as_bytes()).await, Reply::NotFound, "{path}"); + } + assert_eq!(reply_to(b"\x00garbage\r\n\r\n").await, Reply::BadRequest); + } + + #[tokio::test] + async fn an_oversized_head_gets_431_without_reading_past_the_cap() { + let mut long = b"GET /metrics HTTP/1.1\r\nX: ".to_vec(); + long.resize(MAX_HEAD.saturating_mul(2), b'a'); + assert_eq!(reply_to(&long).await, Reply::HeadTooLarge); + let many: String = std::iter::once("GET /metrics HTTP/1.1\r\n".to_owned()) + .chain((0..=MAX_HEADERS).map(|index| format!("H{index}: v\r\n"))) + .chain(std::iter::once("\r\n".to_owned())) + .collect(); + assert_eq!(reply_to(many.as_bytes()).await, Reply::HeadTooLarge); + } + + #[tokio::test] + async fn a_peer_closing_mid_head_is_an_error() { + let mut reader: &[u8] = b"GET /metrics HTTP/1.1\r\n"; + let err = read_request(&mut reader).await.unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof, "{err}"); + } + + #[test] + fn the_tls_config_requires_extended_master_secret() { + praxis_tls::provider::install(); + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("target/tmp/metrics-listener-ems"); + std::fs::create_dir_all(&dir).unwrap(); + let ca = certs::generate_ca("ems test CA").unwrap(); + let leaf = certs::generate_site_cert(&ca, "metrics").unwrap(); + let pair = CertKeyPair { + cert_path: dir.join("tls.crt").display().to_string(), + default: false, + key_path: dir.join("tls.key").display().to_string(), + server_names: Vec::new(), + }; + std::fs::write(&pair.cert_path, &leaf.cert_pem).unwrap(); + std::fs::write(&pair.key_path, &leaf.key_pem).unwrap(); + let config = server_config(ReloadableCertResolver::new(&pair).unwrap()).unwrap(); + assert!(config.require_ems, "TLS 1.2 without EMS is outside SP 800-52r2"); + assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()], "HTTP/1.1 only"); + } + + #[test] + fn a_405_names_get_and_every_reply_closes() { + let (response, _) = Reply::MethodNotAllowed.into_parts(); + assert!(response.starts_with("HTTP/1.1 405 "), "{response}"); + assert!(response.contains("\r\nAllow: GET\r\n"), "{response}"); + assert!(response.contains("\r\nConnection: close\r\n"), "{response}"); + } +} diff --git a/gateway/tests/chart_renders.rs b/gateway/tests/chart_renders.rs new file mode 100644 index 000000000..4d00a1b36 --- /dev/null +++ b/gateway/tests/chart_renders.rs @@ -0,0 +1,116 @@ +//! The chart's default renders load in praxis, so a bad default fails here, not at pod start. +//! +//! Needs `helm` on PATH. Set `GRID_SKIP_CHART_RENDER=1` to skip where helm is absent; +//! otherwise a missing helm fails rather than passing vacuously. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow(clippy::expect_used, clippy::panic, clippy::indexing_slicing, reason = "tests")] +mod tests { + use std::{ + path::PathBuf, + process::Command, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use praxis_core::config::{Config, ConfigFile, DEFAULT_CONFIG}; + + /// Values every case starts from: a rendered gateway config with no auth. + const BASE: &[&str] = &[ + "--set=gatewayConfig.render=true", + "--set=gatewayConfig.localSite=hub", + "--set=gatewayConfig.model=qwen3", + "--set=gatewayConfig.auth.mode=none", + ]; + + /// The default local backend, a local tls workload with its CA, and a remote peer. + const CASES: &[(&str, &[&str])] = &[ + ( + "local plaintext", + &[ + "--set=gatewayConfig.backends[0].cluster=site-a", + "--set=gatewayConfig.backends[0].transport.mode=plaintext", + "--set=gatewayConfig.backends[0].endpoints[0]=10.0.0.5:8000", + ], + ), + ( + "local tls", + &[ + "--set=gatewayConfig.backends[0].cluster=kserve", + "--set=gatewayConfig.backends[0].transport.mode=tls", + "--set=gatewayConfig.backends[0].transport.sni=qwen3.ns.svc", + "--set=gatewayConfig.backends[0].transport.ca.configMap=service-ca", + "--set=gatewayConfig.backends[0].endpoints[0]=10.0.0.6:8443", + ], + ), + ( + "remote peer", + &[ + "--set=gatewayConfig.backends[0].cluster=pool-b", + "--set=gatewayConfig.backends[0].site=site-b", + "--set=gatewayConfig.backends[0].transport.mode=mutual_tls", + "--set=gatewayConfig.backends[0].transport.sni=site-b.grid.internal", + "--set=gatewayConfig.backends[0].endpoints[0]=203.0.113.7:8443", + "--set=tls.enabled=true", + "--set=tls.existingSecret=grid-site-identity", + "--set=tls.caSecret=grid-ca", + ], + ), + ]; + + /// The rendered praxis.yaml for `values`, or `None` when helm is absent and skipping is asked for. + fn render(values: &[&str]) -> Option { + let chart = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../charts/praxis-gateway"); + let output = match Command::new("helm") + .args(["template", "render"]) + .arg(&chart) + .args(BASE) + .args(values) + .args(["--show-only", "templates/gateway-config.yaml"]) + .output() + { + Ok(output) => output, + Err(_) if std::env::var("GRID_SKIP_CHART_RENDER").is_ok_and(|skip| skip == "1") => return None, + Err(error) => panic!("helm is needed to render the chart ({error}); set GRID_SKIP_CHART_RENDER=1 to skip"), + }; + assert!( + output.status.success(), + "helm template: {}", + String::from_utf8_lossy(&output.stderr) + ); + let config_map: serde_yaml::Value = serde_yaml::from_slice(&output.stdout).expect("a ConfigMap"); + Some( + config_map["data"]["praxis.yaml"] + .as_str() + .expect("praxis.yaml") + .to_owned(), + ) + } + + /// Load `yaml` the way the gateway does at startup. + fn load(yaml: &str) -> Result<(), String> { + // Tests run as threads of one process, so each load needs its own file. + static NEXT: AtomicUsize = AtomicUsize::new(0); + let name = format!( + "chart-render-{}-{}.yaml", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ); + let path = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(name); + std::fs::write(&path, yaml).expect("write"); + let file = ConfigFile::read(&path).map_err(|error| error.to_string())?; + let loaded = Config::from_config_file_or(Some(&file), DEFAULT_CONFIG) + .map(drop) + .map_err(|error| error.to_string()); + let _removed = std::fs::remove_file(&path); + loaded + } + + #[test] + fn every_default_render_loads() { + for (name, values) in CASES { + let Some(yaml) = render(values) else { return }; + load(&yaml).unwrap_or_else(|error| panic!("{name}: {error}")); + } + } +} diff --git a/gateway/tests/metrics_listener.rs b/gateway/tests/metrics_listener.rs new file mode 100644 index 000000000..aa8184a92 --- /dev/null +++ b/gateway/tests/metrics_listener.rs @@ -0,0 +1,198 @@ +//! The opt-in metrics listener serves only `GET /metrics` over TLS, to any Host. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::disallowed_methods, + reason = "tests; waits poll a deadline with thread::sleep" +)] +mod tests { + use std::{ + io::{Read as _, Write as _}, + net::{TcpListener, TcpStream}, + path::PathBuf, + process::{Child, Command, Stdio}, + sync::Arc, + thread, + time::{Duration, Instant}, + }; + + use certs::{generate_ca, generate_site_cert}; + use rustls::{ + ClientConfig, ClientConnection, RootCertStore, StreamOwned, + pki_types::{CertificateDer, ServerName, pem::PemObject as _}, + }; + + /// Bound on how long a test waits for the child. + const TIMEOUT: Duration = Duration::from_secs(30); + + /// The SAN `generate_site_cert` gives site `metrics`. + const SERVER_NAME: &str = "metrics.grid.internal"; + + fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port() + } + + /// A running gateway, killed on drop. + struct Gateway(Child); + + impl Drop for Gateway { + fn drop(&mut self) { + drop(self.0.kill()); + drop(self.0.wait()); + } + } + + /// Paths and client trust for one test's cert, key, and config. + struct Setup { + dir: PathBuf, + client: Arc, + listener_port: u16, + } + + fn setup(name: &str) -> Setup { + let dir = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(name); + std::fs::create_dir_all(&dir).unwrap(); + let ca = generate_ca("metrics test CA").unwrap(); + let leaf = generate_site_cert(&ca, "metrics").unwrap(); + std::fs::write(dir.join("tls.crt"), &leaf.cert_pem).unwrap(); + std::fs::write(dir.join("tls.key"), &leaf.key_pem).unwrap(); + let listener_port = free_port(); + std::fs::write( + dir.join("praxis.yaml"), + format!( + "listeners:\n - name: default\n address: \"127.0.0.1:{listener_port}\"\n filter_chains: [main]\n\ + filter_chains:\n - name: main\n filters:\n - filter: static_response\n status: 200\n" + ), + ) + .unwrap(); + let mut roots = RootCertStore::empty(); + roots + .add(CertificateDer::from_pem_slice(ca.cert_pem.as_bytes()).unwrap()) + .unwrap(); + let client = ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(roots) + .with_no_client_auth(); + Setup { + dir, + client: Arc::new(client), + listener_port, + } + } + + fn start(setup: &Setup, metrics_port: u16) -> Gateway { + Gateway( + Command::new(env!("CARGO_BIN_EXE_grid-gateway")) + .arg("--config") + .arg(setup.dir.join("praxis.yaml")) + .env_remove("GRID_SERVING_CONFIG") + .env("GRID_METRICS_ADDR", format!("127.0.0.1:{metrics_port}")) + .env("GRID_METRICS_TLS_CERT", setup.dir.join("tls.crt")) + .env("GRID_METRICS_TLS_KEY", setup.dir.join("tls.key")) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(), + ) + } + + /// Send `head` over TLS and return the full response, or `None` when the connection fails. + fn exchange(client: &Arc, port: u16, head: &str) -> Option { + let stream = TcpStream::connect(("127.0.0.1", port)).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(10))).ok()?; + let name = ServerName::try_from(SERVER_NAME).ok()?; + let conn = ClientConnection::new(Arc::clone(client), name).ok()?; + let mut tls = StreamOwned::new(conn, stream); + tls.write_all(head.as_bytes()).ok()?; + let mut response = String::new(); + tls.read_to_string(&mut response).ok()?; + Some(response) + } + + /// The response to `head`, retrying until the listener is up. + fn eventually(client: &Arc, port: u16, head: &str) -> String { + let deadline = Instant::now().checked_add(TIMEOUT).unwrap(); + loop { + if let Some(response) = exchange(client, port, head) { + return response; + } + assert!(Instant::now() < deadline, "metrics listener never answered on {port}"); + thread::sleep(Duration::from_millis(100)); + } + } + + #[test] + fn only_get_metrics_is_served_and_any_host_is_accepted() { + let setup = setup("metrics-listener-routes"); + let port = free_port(); + let _gateway = start(&setup, port); + + let metrics = eventually( + &setup.client, + port, + "GET /metrics HTTP/1.1\r\nHost: grid-gateway.grid-system.svc\r\n\r\n", + ); + assert!(metrics.starts_with("HTTP/1.1 200 "), "{metrics}"); + assert!(metrics.contains("text/plain; version=0.0.4"), "{metrics}"); + + let kv = eventually(&setup.client, port, "GET /api/kv HTTP/1.1\r\nHost: x\r\n\r\n"); + assert!(kv.starts_with("HTTP/1.1 404 "), "{kv}"); + let wrong_method = eventually( + &setup.client, + port, + "POST /metrics HTTP/1.1\r\nHost: x\r\nContent-Length: 0\r\n\r\n", + ); + assert!(wrong_method.starts_with("HTTP/1.1 405 "), "{wrong_method}"); + } + + #[test] + fn connections_past_the_cap_are_closed_and_slots_come_back() { + let setup = setup("metrics-listener-cap"); + let port = free_port(); + let _gateway = start(&setup, port); + eventually(&setup.client, port, "GET /metrics HTTP/1.1\r\n\r\n"); + + // Four idle connections hold every slot until the handshake deadline. + let held: Vec = std::iter::repeat_with(|| TcpStream::connect(("127.0.0.1", port)).unwrap()) + .take(4) + .collect(); + // Poll inside the 2s handshake deadline, which would free the held slots. + let deadline = Instant::now().checked_add(Duration::from_millis(1500)).unwrap(); + let refused = loop { + if exchange(&setup.client, port, "GET /metrics HTTP/1.1\r\n\r\n").is_none() { + break true; + } + if Instant::now() >= deadline { + break false; + } + thread::sleep(Duration::from_millis(50)); + }; + assert!(refused, "a fifth connection is closed"); + drop(held); + let after = eventually(&setup.client, port, "GET /metrics HTTP/1.1\r\n\r\n"); + assert!(after.starts_with("HTTP/1.1 200 "), "{after}"); + } + + #[test] + fn a_port_praxis_binds_stops_startup() { + let setup = setup("metrics-listener-collision"); + let mut gateway = start(&setup, setup.listener_port); + let deadline = Instant::now().checked_add(TIMEOUT).unwrap(); + let status = loop { + if let Some(status) = gateway.0.try_wait().unwrap() { + break status; + } + assert!( + Instant::now() < deadline, + "the gateway kept running on a colliding port" + ); + thread::sleep(Duration::from_millis(100)); + }; + assert!(!status.success(), "{status}"); + } +} diff --git a/gateway/tests/provider_readiness.rs b/gateway/tests/provider_readiness.rs new file mode 100644 index 000000000..bb6845791 --- /dev/null +++ b/gateway/tests/provider_readiness.rs @@ -0,0 +1,512 @@ +//! A site whose operator reports its provider not ready gets no traffic, and rejoins when it recovers. +//! +//! Time to exclusion is one peer poll after the peer publishes `grid_provider_ready 0`, +//! and time to rejoin one poll after it publishes 1. With the operator's defaults (5s +//! scrape, 2 zero scrapes, 5s poll) that is at most about 15s to exclude and 10s to rejoin. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing, + clippy::min_ident_chars, + clippy::arithmetic_side_effects, + clippy::disallowed_methods, + clippy::too_many_lines, + reason = "tests; waits poll a deadline with thread::sleep" +)] +mod tests { + use std::{ + collections::BTreeMap, + io::{BufRead as _, BufReader, ErrorKind, Read as _, Write as _}, + net::{TcpListener, TcpStream}, + path::{Path, PathBuf}, + process::{Child, Command, Stdio}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU8, Ordering}, + }, + thread, + time::{Duration, Instant}, + }; + + use certs::{CaCert, DEFAULT_TRUST_DOMAIN, GridSpiffeClientVerifier, generate_ca, generate_site_cert}; + use rustls::{ + ServerConfig, ServerConnection, StreamOwned, + pki_types::{CertificateDer, PrivateKeyDer, pem::PemObject as _}, + }; + + /// How often the gateway polls each peer. + const POLL: Duration = Duration::from_millis(200); + /// The stated bound on time to exclusion and to rejoin: one poll, plus slack for a loaded runner. + const BOUND: Duration = Duration::from_millis(3 * 200 + 1_000); + /// The load window, short so a partition outlasts it within the test. + const WINDOW: Duration = Duration::from_millis(1_000); + /// The gap between client requests. + const PACE: Duration = Duration::from_millis(10); + /// How long setup may take. + const DEADLINE: Duration = Duration::from_secs(20); + + const SITES: [&str; 3] = ["site-a", "site-b", "site-d"]; + + /// What a fake peer operator publishes. + const READY: u8 = 0; + const NOT_READY: u8 = 1; + /// Load only, as an operator that does not publish readiness. + const OMITTED: u8 = 2; + /// Accept and drop every poll, as a partitioned link. + const PARTITIONED: u8 = 3; + + /// A peer operator's signals endpoint over mutual TLS, publishing per `mode`. + struct Peer { + addr: String, + mode: Arc, + stop: Arc, + } + + impl Peer { + fn start(ca: &CaCert, site: &'static str) -> Self { + let identity = generate_site_cert(ca, site).expect("peer cert"); + let chain = CertificateDer::pem_slice_iter(identity.cert_pem.as_bytes()) + .collect::, _>>() + .expect("peer chain"); + let key = PrivateKeyDer::from_pem_slice(identity.key_pem.as_bytes()).expect("peer key"); + let provider = rustls::crypto::ring::default_provider(); + let verifier = GridSpiffeClientVerifier::new( + ca.cert_pem.as_bytes(), + DEFAULT_TRUST_DOMAIN, + provider.signature_verification_algorithms, + ) + .expect("client verifier"); + let config = Arc::new( + ServerConfig::builder_with_provider(Arc::new(provider)) + .with_safe_default_protocol_versions() + .expect("versions") + .with_client_cert_verifier(verifier) + .with_single_cert(chain, key) + .expect("server config"), + ); + let listener = TcpListener::bind("127.0.0.1:0").expect("bind peer"); + let addr = listener.local_addr().expect("peer addr").to_string(); + let mode = Arc::new(AtomicU8::new(READY)); + let stop = Arc::new(AtomicBool::new(false)); + let (serving, stopping) = (Arc::clone(&mode), Arc::clone(&stop)); + thread::spawn(move || { + for stream in listener.incoming() { + if stopping.load(Ordering::SeqCst) { + return; + } + let Ok(stream) = stream else { continue }; + let publishing = serving.load(Ordering::SeqCst); + if publishing == PARTITIONED { + continue; + } + let _bounded = stream.set_read_timeout(Some(Duration::from_secs(1))); + let Ok(conn) = ServerConnection::new(Arc::clone(&config)) else { + continue; + }; + let mut tls = StreamOwned::new(conn, stream); + let mut request = Vec::new(); + let mut buf = [0_u8; 1024]; + while !request.windows(4).any(|w| w == b"\r\n\r\n") { + match tls.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => request.extend_from_slice(&buf[..n]), + } + } + let body = signals(site, publishing); + // Samples stamped at the Date header read as age zero. + let response = format!( + "HTTP/1.1 200 OK\r\nDate: Thu, 01 Jan 1970 00:00:01 GMT\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _sent = tls.write_all(response.as_bytes()).and_then(|()| tls.flush()); + } + }); + Self { addr, mode, stop } + } + } + + impl Drop for Peer { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + let _woken = TcpStream::connect(&self.addr); + } + } + + /// What `site`'s operator publishes in `mode`: an idle queue, and its readiness verdict. + fn signals(site: &str, mode: u8) -> String { + let labels = format!(r#"grid_site="{site}",grid_provider="pool-{site}""#); + let load = format!("llm_d_epp_average_queue_size{{{labels}}} 0 1000\n"); + match mode { + OMITTED => load, + _ => format!( + "{load}grid_provider_ready{{{labels}}} {} 1000\n", + u8::from(mode == READY) + ), + } + } + + /// A plain HTTP backend answering 200 with its site name. + fn backend(site: &'static str) -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind backend"); + let port = listener.local_addr().expect("backend addr").port(); + thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { continue }; + thread::spawn(move || { + let mut head = Vec::new(); + let mut buf = [0_u8; 1024]; + while !head.windows(4).any(|w| w == b"\r\n\r\n") { + match stream.read(&mut buf) { + Ok(0) => return, + Ok(n) => head.extend_from_slice(&buf[..n]), + Err(e) if e.kind() == ErrorKind::Interrupted => {}, + Err(_) => return, + } + } + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{site}", + site.len() + ); + let _sent = stream.write_all(response.as_bytes()); + }); + } + }); + port + } + + fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind") + .local_addr() + .expect("addr") + .port() + } + + fn drain(pipe: impl std::io::Read + Send + 'static, lines: Arc>>) { + thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + lines.lock().expect("output").push(line); + } + }); + } + + struct Gateway { + child: Child, + output: Arc>>, + listen: u16, + admin: u16, + } + + impl Gateway { + fn start(work: &Path, backends: &[(&str, u16)], peers: &[(&str, &Peer)], identity: &Path) -> Self { + let (listen, admin) = (free_port(), free_port()); + let clusters = |indent: &str| -> String { + backends + .iter() + .map(|(site, port)| { + format!("{indent}- name: \"pool-{site}\"\n{indent} endpoints: [\"127.0.0.1:{port}\"]\n") + }) + .collect() + }; + let config = work.join("praxis.yaml"); + std::fs::write( + &config, + format!( + "admin:\n address: \"127.0.0.1:{admin}\"\n# The test backends are on loopback.\ninsecure_options:\n allow_private_endpoints: true\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nclusters:\n{top}filter_chains:\n - name: main\n filters:\n - filter: grid_site_route\n model_header: X-Gateway-Model-Name\n - filter: load_balancer\n clusters:\n{inline}", + top = clusters(" "), + inline = clusters(" "), + ), + ) + .expect("praxis config"); + let path = |key: &str| identity.join(key).to_string_lossy().into_owned(); + let candidates: Vec = SITES + .iter() + .map(|site| { + format!(r#"{{"kind":"inference_model","name":"llama","site":"{site}","cluster":"pool-{site}"}}"#) + }) + .collect(); + let peers: Vec = peers + .iter() + .map(|(site, peer)| { + format!( + r#"{{"site":"{site}","addr":"{addr}","server_name":"{site}.grid.internal","authority":"{site}.grid.internal","interval_ms":{poll},"connect_timeout_ms":1000,"request_timeout_ms":1000,"grid_ca_path":"{ca}","client_cert_path":"{cert}","client_key_path":"{key}"}}"#, + addr = peer.addr, + poll = POLL.as_millis(), + ca = path("ca.crt"), + cert = path("tls.crt"), + key = path("tls.key"), + ) + }) + .collect(); + let serving = work.join("serving-config.json"); + std::fs::write( + &serving, + format!( + r#"{{"local_site":"hub","window_secs":60,"load_window_ms":{window},"candidates":[{}],"peers":[{}]}}"#, + candidates.join(","), + peers.join(","), + window = WINDOW.as_millis(), + ), + ) + .expect("serving config"); + let mut child = Command::new(env!("CARGO_BIN_EXE_grid-gateway")) + .arg("--config") + .arg(&config) + .env("GRID_SERVING_CONFIG", &serving) + .env("RUST_LOG", "info") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn gateway"); + let output = Arc::new(Mutex::new(Vec::new())); + drain(child.stdout.take().expect("stdout"), Arc::clone(&output)); + drain(child.stderr.take().expect("stderr"), Arc::clone(&output)); + Self { + child, + output, + listen, + admin, + } + } + + /// One request for `llama`: its status, the site that served it, and Retry-After. + /// The admin listener's Prometheus exposition. + fn metrics(&self) -> String { + let mut stream = TcpStream::connect(("127.0.0.1", self.admin)).expect("connect admin"); + write!( + stream, + "GET /metrics HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("send"); + let mut response = String::new(); + let _read = stream.read_to_string(&mut response); + response + } + + fn request(&self) -> Option<(u16, String, Option)> { + let (head, body) = self.exchange()?; + let status = head.split(' ').nth(1)?.parse().ok()?; + let retry_after = head + .lines() + .find_map(|l| { + l.strip_prefix("retry-after: ") + .or_else(|| l.strip_prefix("Retry-After: ")) + }) + .map(str::to_owned); + Some((status, body, retry_after)) + } + + /// One request for llama: the response head and body. + fn exchange(&self) -> Option<(String, String)> { + let mut stream = TcpStream::connect(("127.0.0.1", self.listen)).ok()?; + stream.set_read_timeout(Some(Duration::from_secs(5))).ok()?; + stream + .write_all( + b"GET /v1/models HTTP/1.1\r\nHost: gw\r\nX-Gateway-Model-Name: llama\r\nConnection: close\r\n\r\n", + ) + .ok()?; + let mut response = String::new(); + stream.read_to_string(&mut response).ok()?; + let (head, body) = response.split_once("\r\n\r\n")?; + Some((head.to_owned(), body.to_owned())) + } + + fn eventually(&self, what: &str, mut done: impl FnMut() -> bool) { + let deadline = Instant::now() + DEADLINE; + while !done() { + if Instant::now() > deadline { + let output = self.output.lock().expect("output").join("\n"); + panic!("timed out waiting for {what}; gateway output:\n{output}"); + } + thread::sleep(Duration::from_millis(50)); + } + } + } + + impl Drop for Gateway { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + + /// Requests per serving site over `n` paced requests; any non-200 counts under `error`. + fn split(gateway: &Gateway, n: usize) -> BTreeMap { + let mut counts = BTreeMap::new(); + for _ in 0..n { + let key = match gateway.request() { + Some((200, site, _)) => site, + _ => "error".to_owned(), + }; + *counts.entry(key).or_default() += 1; + thread::sleep(PACE); + } + counts + } + + /// Paced requests until twenty in a row are served by a site `pred` accepts, and how long + /// until the first of them. + /// + /// Twenty, since sites are drawn at random: with one rejected site of three still drawn, + /// twenty accepted in a row has probability (2/3)^20, below 1 in 3000. + fn time_until(gateway: &Gateway, mut pred: impl FnMut(&str) -> bool) -> Duration { + let start = Instant::now(); + let mut streak = 0; + let mut streak_start = start; + while streak < 20 { + assert!(start.elapsed() < DEADLINE, "never settled"); + let sent = Instant::now(); + match gateway.request() { + Some((200, site, _)) if pred(&site) => { + if streak == 0 { + streak_start = sent; + } + streak += 1; + }, + _ => streak = 0, + } + thread::sleep(PACE); + } + streak_start.duration_since(start) + } + + #[test] + fn site_b_not_ready_routes_to_the_other_sites_and_rejoins_when_ready() { + let work = + PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("provider-readiness-{}", std::process::id())); + let _cleared = std::fs::remove_dir_all(&work); + let identity = work.join("identity"); + std::fs::create_dir_all(&identity).expect("identity dir"); + let ca = generate_ca("grid-ca").expect("ca"); + let hub = generate_site_cert(&ca, "hub").expect("hub cert"); + for (key, pem) in [ + ("ca.crt", &ca.cert_pem), + ("tls.crt", &hub.cert_pem), + ("tls.key", &hub.key_pem), + ] { + std::fs::write(identity.join(key), pem).expect("identity file"); + } + let peers: Vec<(&str, Peer)> = SITES.iter().map(|site| (*site, Peer::start(&ca, site))).collect(); + let backends: Vec<(&str, u16)> = SITES.iter().map(|site| (*site, backend(site))).collect(); + let gateway = Gateway::start( + &work, + &backends, + &peers.iter().map(|(site, peer)| (*site, peer)).collect::>(), + &identity, + ); + let site_b = &peers[1].1; + + gateway.eventually("every site serving", || { + let counts = split(&gateway, 9); + SITES.iter().all(|site| counts.contains_key(*site)) + }); + // site-b's provider goes not ready. + site_b.mode.store(NOT_READY, Ordering::SeqCst); + let to_exclusion = time_until(&gateway, |site| site != "site-b"); + assert!( + to_exclusion <= BOUND, + "excluded after {to_exclusion:?}, bound {BOUND:?}" + ); + let excluded = split(&gateway, 200); + assert_eq!( + excluded.get("error"), + None, + "no client errors once excluded: {excluded:?}" + ); + assert_eq!(excluded.get("site-b"), None, "nothing to site-b: {excluded:?}"); + assert!( + // Each takes about 100 of 200 at random, standard deviation about 7. + excluded.get("site-a").copied().unwrap_or(0) >= 70 && excluded.get("site-d").copied().unwrap_or(0) >= 70, + "site-a and site-d share the load: {excluded:?}" + ); + + // Its link partitions past the load window: the last verdict still stands. + site_b.mode.store(PARTITIONED, Ordering::SeqCst); + thread::sleep(WINDOW * 3); + let partitioned = split(&gateway, 50); + assert_eq!( + partitioned.get("site-b"), + None, + "a silent peer is not readmitted: {partitioned:?}" + ); + assert_eq!(partitioned.get("error"), None, "{partitioned:?}"); + + // It heals with an operator that no longer publishes readiness: absent reads as ready. + site_b.mode.store(OMITTED, Ordering::SeqCst); + let healed = Instant::now(); + while !gateway + .request() + .is_some_and(|(status, site, _)| status == 200 && site == "site-b") + { + assert!(healed.elapsed() < DEADLINE, "site-b never rejoined without the series"); + thread::sleep(PACE); + } + assert!( + healed.elapsed() <= BOUND, + "rejoined without the series after {:?}", + healed.elapsed() + ); + + // Back to publishing, not ready, then ready. + site_b.mode.store(NOT_READY, Ordering::SeqCst); + time_until(&gateway, |site| site != "site-b"); + site_b.mode.store(READY, Ordering::SeqCst); + let start = Instant::now(); + while !gateway + .request() + .is_some_and(|(status, site, _)| status == 200 && site == "site-b") + { + assert!(start.elapsed() < DEADLINE, "site-b never rejoined"); + thread::sleep(PACE); + } + let to_rejoin = start.elapsed(); + assert!(to_rejoin <= BOUND, "rejoined after {to_rejoin:?}, bound {BOUND:?}"); + let rejoined = split(&gateway, 300); + assert_eq!(rejoined.get("error"), None, "{rejoined:?}"); + // Each takes about 100 of 300 at random, standard deviation about 8. + for site in SITES { + let share = rejoined.get(site).copied().unwrap_or(0); + assert!( + (70..=130).contains(&share), + "{site} took {share} of 300, a herd: {rejoined:?}" + ); + } + + // Every site not ready: a known model answers 503 with Retry-After, not 404. + for (_, peer) in &peers { + peer.mode.store(NOT_READY, Ordering::SeqCst); + } + gateway.eventually("503 once every site is excluded", || { + gateway.request().is_some_and(|(status, _, after)| { + status == 503 + && after + .and_then(|a| a.parse::().ok()) + .is_some_and(|a| (3..=7).contains(&a)) + }) + }); + + let metrics = gateway.metrics(); + let counted = |prefix: &str| -> f64 { + metrics + .lines() + .filter(|line| line.starts_with(prefix)) + .filter_map(|line| line.rsplit_once(' ').and_then(|(_, value)| value.parse::().ok())) + .sum() + }; + for site in SITES { + let routed = counted(&format!("grid_route_decisions_total{{site=\"{site}\",")); + assert!(routed > 0.0, "no routing decisions for {site}:\n{metrics}"); + } + let refused = counted("grid_route_decisions_total{site=\"\",reason=\"not_ready\"}"); + assert!(refused > 0.0, "the 503s are not counted:\n{metrics}"); + let score = "grid_route_site_score{site=\"site-b\",cluster=\"pool-site-b\"} NaN"; + assert!(metrics.contains(score), "an excluded site keeps a score:\n{metrics}"); + + drop(gateway); + let _cleaned = std::fs::remove_dir_all(&work); + } +} diff --git a/gateway/tests/unrouted.rs b/gateway/tests/unrouted.rs new file mode 100644 index 000000000..5b3f3320c --- /dev/null +++ b/gateway/tests/unrouted.rs @@ -0,0 +1,218 @@ +//! A request the grid router cannot route is answered by the router and logged below WARN. + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::expect_used, + clippy::panic, + clippy::disallowed_methods, + clippy::arithmetic_side_effects, + clippy::too_many_lines, + reason = "tests; waits poll a deadline with thread::sleep" +)] +mod tests { + use std::{ + io::{BufRead as _, BufReader, Read as _, Write as _}, + net::{TcpListener, TcpStream}, + path::PathBuf, + process::{Child, Command, Stdio}, + sync::{Arc, Mutex}, + thread, + time::{Duration, Instant}, + }; + + /// How long the gateway may take to start listening. + const DEADLINE: Duration = Duration::from_secs(20); + + fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind") + .local_addr() + .expect("addr") + .port() + } + + /// Collect `pipe`'s lines into `lines` on a background thread. + fn drain(pipe: impl std::io::Read + Send + 'static, lines: Arc>>) { + thread::spawn(move || { + for line in BufReader::new(pipe).lines().map_while(Result::ok) { + lines.lock().expect("output").push(line); + } + }); + } + + /// The gateway with its output collected in the background. + struct Gateway { + child: Child, + output: Arc>>, + listen: u16, + admin: u16, + } + + impl Gateway { + /// Route `llama` to a local cluster whose only endpoint refuses connections. + fn start(work: &std::path::Path) -> Self { + std::fs::create_dir_all(work).expect("work dir"); + let (listen, admin, dead) = (free_port(), free_port(), free_port()); + let serving = work.join("serving-config.json"); + std::fs::write( + &serving, + r#"{"local_site":"local","window_secs":60,"load_window_ms":30000,"candidates":[{"kind":"inference_model","name":"llama","site":"local","cluster":"pool-local"}],"peers":[]}"#, + ) + .expect("serving config"); + let config = work.join("praxis.yaml"); + std::fs::write( + &config, + format!( + "insecure_options:\n allow_private_endpoints: true\nadmin:\n address: \"127.0.0.1:{admin}\"\nlisteners:\n - name: default\n address: \"127.0.0.1:{listen}\"\n filter_chains: [main]\nfilter_chains:\n - name: main\n filters:\n - filter: grid_site_route\n model_header: X-Model\n - filter: load_balancer\n clusters:\n - name: pool-local\n endpoints: [\"127.0.0.1:{dead}\"]\n" + ), + ) + .expect("praxis config"); + let mut child = Command::new(env!("CARGO_BIN_EXE_grid-gateway")) + .arg("--config") + .arg(&config) + .env("GRID_SERVING_CONFIG", &serving) + .env("RUST_LOG", "debug") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn gateway"); + let output = Arc::new(Mutex::new(Vec::new())); + drain(child.stdout.take().expect("stdout"), Arc::clone(&output)); + drain(child.stderr.take().expect("stderr"), Arc::clone(&output)); + let mut gateway = Self { + child, + output, + listen, + admin, + }; + let deadline = Instant::now() + DEADLINE; + while TcpStream::connect(("127.0.0.1", listen)).is_err() { + assert!( + Instant::now() < deadline && gateway.child.try_wait().ok().flatten().is_none(), + "gateway never listened; output:\n{}", + gateway.output() + ); + thread::sleep(Duration::from_millis(50)); + } + gateway + } + + fn output(&self) -> String { + self.output.lock().expect("output").join("\n") + } + + fn lines(&self) -> usize { + self.output.lock().expect("output").len() + } + + /// The admin listener's Prometheus exposition. + fn metrics(&self) -> String { + let mut stream = TcpStream::connect(("127.0.0.1", self.admin)).expect("connect admin"); + write!( + stream, + "GET /metrics HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n" + ) + .expect("send"); + let mut response = String::new(); + let _read = stream.read_to_string(&mut response); + response + } + + /// The status code of a GET for `path` with `headers`. + fn status(&self, path: &str, headers: &str) -> u16 { + let mut stream = TcpStream::connect(("127.0.0.1", self.listen)).expect("connect"); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("read timeout"); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: grid.example\r\n{headers}Connection: close\r\n\r\n" + ) + .expect("send"); + let mut response = String::new(); + let _read = stream.read_to_string(&mut response); + response + .split_whitespace() + .nth(1) + .and_then(|code| code.parse().ok()) + .unwrap_or_else(|| panic!("no status line in {response:?}")) + } + } + + impl Drop for Gateway { + fn drop(&mut self) { + let _killed = self.child.kill(); + let _reaped = self.child.wait(); + } + } + + /// Whether a log line is at WARN or ERROR. + fn loud(line: &str) -> bool { + line.split_whitespace() + .take(3) + .any(|word| word.contains("WARN") || word.contains("ERROR")) + } + + #[test] + fn an_unrouted_request_is_answered_by_the_router_below_warn() { + let work = PathBuf::from(env!("CARGO_TARGET_TMPDIR")).join(format!("unrouted-{}", std::process::id())); + let _cleared = std::fs::remove_dir_all(&work); + let gateway = Gateway::start(&work); + let before = gateway.lines(); + + assert_eq!(gateway.status("/health", ""), 400, "no model in the request"); + assert_eq!( + gateway.status("/v1/models", "X-Model: granite\r\n"), + 404, + "a model with no admitted candidate" + ); + + let deadline = Instant::now() + Duration::from_secs(5); + while !gateway.output().contains("grid_site_route: no model in the request") { + assert!( + Instant::now() < deadline, + "no debug line; output:\n{}", + gateway.output() + ); + thread::sleep(Duration::from_millis(50)); + } + let metrics = gateway.metrics(); + // Unmeasured, so the order ranks it last with an infinite score. + let score = "grid_route_site_score{site=\"local\",cluster=\"pool-local\"} inf"; + assert!(metrics.contains(score), "no {score:?} in:\n{metrics}"); + // No model and an unknown model both count as bad_request. + let refused = "grid_route_decisions_total{site=\"\",reason=\"bad_request\"} 2"; + assert!(metrics.contains(refused), "no {refused:?} in:\n{metrics}"); + // Labels come from the serving config and a closed set, never from the request. + for series in metrics + .lines() + .filter(|line| line.starts_with("grid_route_decisions_total{")) + { + let label = |name: &str| { + series + .split_once(&format!("{name}=\"")) + .and_then(|(_, rest)| rest.split_once('"')) + .map_or_else(|| panic!("no {name} in {series}"), |(value, _)| value.to_owned()) + }; + assert!( + ["", "local"].contains(&label("site").as_str()), + "request-derived site: {series}" + ); + assert!( + ["routed", "fallback", "not_ready", "no_route", "bad_request", "shed"] + .contains(&label("reason").as_str()), + "reason outside the closed set: {series}" + ); + } + let output = gateway.output.lock().expect("output").clone(); + let loud_lines: Vec<&String> = output.iter().skip(before).filter(|line| loud(line)).collect(); + assert!( + loud_lines.is_empty(), + "an unrouted request logged at WARN or ERROR: {loud_lines:#?}" + ); + + drop(gateway); + let _cleaned = std::fs::remove_dir_all(&work); + } +} diff --git a/mock-providers/src/openai.rs b/mock-providers/src/openai.rs index 04886a6f6..57bdb354e 100644 --- a/mock-providers/src/openai.rs +++ b/mock-providers/src/openai.rs @@ -159,7 +159,11 @@ async fn metrics(axum::extract::State(state): axum::extract::State) -> Response::builder() .status(StatusCode::OK) .header(http::header::CONTENT_TYPE, "text/plain; version=0.0.4") - .body(Body::from(format!("grid_demo_queue_depth {}\n", state.queue_depth))) + // A ready count, as an EPP exposes, so a grid operator reads the mock as ready. + .body(Body::from(format!( + "grid_demo_queue_depth {}\ninference_pool_ready_pods 1\n", + state.queue_depth + ))) .unwrap_or_default() } @@ -448,7 +452,10 @@ mod tests { let resp = send(req).await; assert_eq!(resp.status(), StatusCode::OK); let body = to_bytes(resp.into_body(), 4096).await.unwrap_or_default(); - assert_eq!(body.as_ref(), b"grid_demo_queue_depth 0.25\n"); + assert_eq!( + body.as_ref(), + b"grid_demo_queue_depth 0.25\ninference_pool_ready_pods 1\n" + ); } #[tokio::test] diff --git a/operator/src/controller/grid_network.rs b/operator/src/controller/grid_network.rs index 9438805cd..5339a31d8 100644 --- a/operator/src/controller/grid_network.rs +++ b/operator/src/controller/grid_network.rs @@ -34,6 +34,7 @@ use crate::{ inference_provider::InferenceProvider, }, error::OperatorError, + readiness, resources::{ consumer_config::{self, ConsumerConfigError}, overlay_envelope, provider_admission, provider_metrics, routing_overlay, secret, @@ -98,6 +99,9 @@ pub struct OperatorCtx { /// after async DNS resolution and the SWIM channel announcement. pub(crate) last_seeds: std::sync::Mutex>>, + /// Sites each `GridNetwork`'s serving config last refused, for warning on change. + pub(crate) refused_sites: serving_config::RefusedSites, + /// Who may read the signals endpoint, keyed by presented-cert fingerprint. /// Set by reconcile from each `GridSite`'s trust pins. pub(crate) peer_identities: signals::PeerIdentities, @@ -112,6 +116,12 @@ pub struct OperatorCtx { /// Served-model sets discovered from this site's providers. pub(crate) served_models: served_models::ServedModelStore, + /// Each provider's latest scrape, from which its readiness is resolved. + pub(crate) readiness: readiness::ReadinessStore, + + /// How often the signals loop scrapes providers; a readiness window is at least two. + scrape_interval: Duration, + /// Signal transport resolved once at startup: gossip or poll. /// /// Under poll the operator stops carrying metrics in gossip and scoring the @@ -271,10 +281,13 @@ impl OperatorCtx { metrics_cache: Mutex::new(provider_metrics::MetricsCache::new()), admission_memory: Mutex::new(provider_admission::AdmissionMemory::default()), last_seeds: std::sync::Mutex::new(HashMap::new()), + refused_sites: std::sync::Mutex::new(HashMap::new()), peer_identities: signals::PeerIdentities::new(), peers: signals::SignalStore::new(), signals: signals::SignalStore::new(), served_models: served_models::ServedModelStore::new(), + readiness: readiness::ReadinessStore::default(), + scrape_interval: Duration::ZERO, signal_mode, serving_writes: WriteGate::default(), peer_settings: PeerSettings::default(), @@ -357,6 +370,13 @@ impl OperatorCtx { self.site_name.as_deref() } + /// Record how often the signals loop scrapes providers. + #[must_use] + pub const fn with_scrape_interval(mut self, interval: Duration) -> Self { + self.scrape_interval = interval; + self + } + /// Name the certificate the operator issues itself after this site, not the network. #[must_use] pub fn with_site_name(mut self, site_name: Option) -> Self { @@ -419,11 +439,262 @@ pub async fn refresh_signals(ctx: &OperatorCtx, client: &Client, network_name: & let providers = list_all_inference_providers(client).await?; ctx.signals.set_access(signal_access(&providers)); Box::pin(register_peers(ctx, client)).await?; - let collected = provider_metrics::collect_provider_signals(network_name, &providers, Some(client)).await; + Box::pin(provider_metrics::collect_provider_signals( + network_name, + &providers, + Some(client), + &ctx.readiness, + )) + .await; + let collected = Box::pin(resolve_readiness(ctx, client, &providers, network_name)).await; publish_signals(ctx, collected); Ok(()) } +/// Resolve each provider's readiness, write its `Ready` condition, and return what to publish. +async fn resolve_readiness( + ctx: &OperatorCtx, + client: &Client, + providers: &[InferenceProvider], + network_name: &str, +) -> HashMap> { + ctx.readiness.retain( + &providers + .iter() + .filter_map(|p| { + provider_metrics::signal_scrape_plan(p) + .map(|plan| readiness::key(&p.spec.grid_network_ref, plan.identity)) + }) + .collect(), + ); + let now = Instant::now(); + let mut collected = HashMap::new(); + for provider in providers.iter().filter(|p| p.spec.grid_network_ref == network_name) { + let Some(entry) = readiness_entry(ctx, provider, now) else { + continue; + }; + if let Err(error) = Box::pin(apply_ready_condition(client, provider, &entry.verdict)).await { + tracing::warn!(provider = entry.identity, %error, "provider Ready condition not written"); + } + if let Some(published) = entry.published { + collected.insert(entry.identity.to_owned(), published); + } + } + collected +} + +/// One provider's readiness and the signals to publish for it. +struct ReadinessEntry<'provider> { + /// Routing identity. + identity: &'provider str, + /// The verdict. + verdict: readiness::Verdict, + /// Signals to publish, `None` for a provider without metrics: absent reads as ready. + published: Option>, +} + +/// A provider's readiness entry, `None` for no identity or not yet judged. +/// +/// A provider not scraped for readiness, with no metrics config or no usable +/// endpoint, is `Unknown` and publishes nothing: absent reads as ready. +fn readiness_entry<'provider>( + ctx: &OperatorCtx, + provider: &'provider InferenceProvider, + now: Instant, +) -> Option> { + let identity = routing_overlay::routing_identity(provider)?; + let key = readiness::key(&provider.spec.grid_network_ref, identity); + let capacity = capacity_sample(provider, ctx.readiness.ready_endpoints(&key)); + if provider_metrics::signal_scrape_plan(provider).is_none() { + return Some(ReadinessEntry { + identity, + verdict: readiness::Verdict { + reason: readiness::Reason::MetricsNotConfigured, + message: "no metricsConfig endpoint to read readiness from".to_owned(), + }, + // With no scrape there are no ready endpoints to multiply, so capacity is unknown. + published: None, + }); + } + let verdict = provider_readiness(ctx, provider, now)?; + let ready = ready_sample(&verdict); + // A fresh scrape republishes with the verdict. Without one, the last published + // entry ages on its own, unless the verdict turned not ready. + let fresh = ctx.readiness.take_fresh(&key); + Some(ReadinessEntry { + identity, + published: published_signals(fresh, ready, capacity, verdict.reason.excludes()), + verdict, + }) +} + +/// What to publish for a scraped provider: a fresh scrape with the verdict and capacity, +/// or only those when it is not ready. Without either, the last entry ages on its own. +fn published_signals( + fresh: Option>, + ready: signals::Observation, + capacity: Option, + excluded: bool, +) -> Option> { + match fresh { + Some(mut held) => { + held.push(ready); + held.extend(saturation_sample(&held, capacity.as_ref())); + held.extend(capacity); + Some(held) + }, + None if excluded => Some(std::iter::once(ready).chain(capacity).collect()), + None => None, + } +} + +/// The `grid_provider_saturation_ratio` sample: requests held in `held` over `capacity`, `None` +/// when either is unknown. +fn saturation_sample( + held: &[signals::Observation], + capacity: Option<&signals::Observation>, +) -> Option { + let in_flight = held.iter().find(|o| o.metric == readiness::IN_FLIGHT_SIGNAL)?.value; + let capacity = capacity?.value; + // A non-finite or negative count from a misbehaving EPP publishes no ratio, never inf. + (capacity > 0.0 && in_flight.is_finite() && in_flight >= 0.0).then(|| signals::Observation { + metric: readiness::SATURATION_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value: in_flight / capacity, + timestamp_ms: None, + }) +} + +/// The `grid_provider_capacity_requests` sample: the provider's per-endpoint `maxRunning` times its +/// `ready` endpoints. `None` when it declares none or no endpoint is fresh, which leaves +/// capacity unknown rather than zero. +fn capacity_sample(provider: &InferenceProvider, ready: Option) -> Option { + let max_running = provider.spec.max_running?; + let ready = ready.filter(|ready| *ready > 0.0)?; + Some(signals::Observation { + metric: readiness::CAPACITY_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value: f64::from(max_running) * ready, + timestamp_ms: None, + }) +} + +/// The `grid_provider_ready` sample for `verdict`: 1 when it serves, 0 when not. +fn ready_sample(verdict: &readiness::Verdict) -> signals::Observation { + signals::Observation { + metric: readiness::READY_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value: if verdict.reason.excludes() { 0.0 } else { 1.0 }, + timestamp_ms: None, + } +} + +/// How long a provider's last good scrape stands. +/// +/// `staleMetricsSeconds`, else half the signal TTL, and never under two scrape +/// intervals, so a provider is not judged stale between successful scrapes. +fn readiness_stale_after(ctx: &OperatorCtx, provider: &InferenceProvider) -> Duration { + provider + .spec + .metrics_config + .as_ref() + .and_then(|mc| mc.stale_metrics_seconds) + .filter(|secs| *secs > 0) + .map_or_else(|| site_signals_ttl() / 2, |secs| Duration::from_secs(secs.into())) + .max(ctx.scrape_interval.saturating_mul(2)) +} + +/// The provider's readiness verdict, `None` when it is not scraped for readiness or not yet judged. +pub(crate) fn provider_readiness( + ctx: &OperatorCtx, + provider: &InferenceProvider, + now: Instant, +) -> Option { + let plan = provider_metrics::signal_scrape_plan(provider)?; + let unavailable = provider + .status + .as_ref() + .is_some_and(|status| status.phase == crate::crd::inference_provider::ProviderPhase::Unavailable); + ctx.readiness.verdict( + &readiness::key(&provider.spec.grid_network_ref, plan.identity), + unavailable, + readiness_stale_after(ctx, provider), + now, + ) +} + +/// Write the provider's `Ready` condition when its status or reason changed. +async fn apply_ready_condition( + client: &Client, + provider: &InferenceProvider, + verdict: &readiness::Verdict, +) -> Result<(), OperatorError> { + let Some((name, patch)) = ready_condition_patch(provider, verdict) else { + return Ok(()); + }; + let api: Api = Api::all(client.clone()); + Box::pin(api.patch_status( + name, + &PatchParams::apply(READINESS_FIELD_MANAGER).force(), + &Patch::Apply(patch), + )) + .await?; + // A turn away from Ready warns; a return to Ready or a wait is informational. + if verdict.reason.excludes() { + tracing::warn!( + provider = name, + status = verdict.reason.status(), + reason = verdict.reason.as_str(), + message = %verdict.message, + "provider readiness changed" + ); + } else { + tracing::info!( + provider = name, + status = verdict.reason.status(), + reason = verdict.reason.as_str(), + message = %verdict.message, + "provider readiness changed" + ); + } + Ok(()) +} + +/// The status patch carrying the provider's new `Ready` condition, `None` when unchanged. +fn ready_condition_patch<'provider>( + provider: &'provider InferenceProvider, + verdict: &readiness::Verdict, +) -> Option<(&'provider str, serde_json::Value)> { + let name = provider.metadata.name.as_deref()?; + let status = provider.status.as_ref(); + let current = status.map_or(&[][..], |status| status.conditions.as_slice()); + let shown = verdict.reason.display(); + let now = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .ok()?; + // Apply always carries both fields this manager owns: one it left out would be removed. + let condition = match readiness::ready_condition(current, verdict, &now, provider.metadata.generation) { + Some(condition) => condition, + None if status.and_then(|status| status.state.as_deref()) == Some(shown) => return None, + None => current + .iter() + .find(|condition| condition.type_ == readiness::READY_CONDITION)? + .clone(), + }; + Some(( + name, + serde_json::json!({ + "apiVersion": "grid.praxis.fast/v1alpha1", + "kind": "InferenceProvider", + "metadata": { "name": name }, + "status": { "conditions": [condition], "state": shown } + }), + )) +} + +/// Server-side apply manager for the `Ready` condition, apart from provider reconciliation. +const READINESS_FIELD_MANAGER: &str = "grid-operator-readiness"; + /// Poll this site's providers once and hold the models they serve. /// /// Called from its own loop rather than from reconcile, like @@ -939,6 +1210,13 @@ pub async fn reconcile(network: Arc, ctx: Arc) -> Resu memory.evaluate(&memory_key, observation, admission_policy, now), provider.spec.traffic_policy.as_ref().is_some_and(|p| p.drain), ); + // Readiness gates admission whatever the scoring strategy, NoMetrics included. + let not_ready = provider_readiness(&ctx, provider, now).is_some_and(|verdict| verdict.reason.excludes()); + let state = if not_ready { + crate::resources::geography::AdmissionState::Excluded + } else { + state + }; admission_keys.push(memory_key); admission_states.insert(identity, state); } @@ -1810,6 +2088,10 @@ struct ServingSource<'src> { gate: &'src WriteGate, /// Peer addressing resolved at startup. settings: &'src PeerSettings, + /// How often this operator scrapes its providers. + scrape_interval: Duration, + /// Sites each network's serving config last refused. + refused: &'src serving_config::RefusedSites, } /// Build the serving source from membership, `None` outside poll mode. @@ -1841,6 +2123,8 @@ fn serving_source<'src>( pins, gate: &ctx.serving_writes, settings: &ctx.peer_settings, + scrape_interval: ctx.scrape_interval, + refused: &ctx.refused_sites, }) } @@ -1860,6 +2144,7 @@ fn render_serving_text( tls_mount, local_signals_addr: source.settings.local_signals_addr.as_deref(), pins: &source.pins, + scrape_interval: source.scrape_interval, }; let members = source.members.iter().map(|(site, endpoint)| (*site, endpoint.as_str())); serving_config::render(overlay, members, &inputs) @@ -1877,6 +2162,7 @@ async fn apply_serving_config( gw_ref: &GatewayRef, client: &Client, ) -> Result, OperatorError> { + serving_config::warn_refused_sites(overlay, network_name, source.refused); let Some(text) = render_serving_text(overlay, source, gw_ref)? else { tracing::debug!(gateway = %gw_ref.name, "serving config has no candidates; leaving any prior config"); return Ok(None); @@ -3776,6 +4062,139 @@ fn parse_metrics_refresh_interval(value: &str) -> Result InferenceProvider { + serde_json::from_value(serde_json::json!({ + "apiVersion": "grid.praxis.fast/v1alpha1", + "kind": "InferenceProvider", + "metadata": { "name": "qwen3-site-b", "generation": 1 }, + "spec": { + "gridNetworkRef": "grid", + "providerKind": "self_hosted", + "backendKind": "local", + "endpoint": "http://localhost:8000", + "models": [] + }, + "status": status + })) + .expect("provider") + } + + fn ready_condition_json(status: &str, reason: &str) -> serde_json::Value { + serde_json::json!({ + "type": "Ready", "status": status, "reason": reason, "message": "m", + "lastTransitionTime": "2026-10-03T00:00:00Z", "observedGeneration": 1 + }) + } + + #[test] + #[expect( + clippy::expect_used, + clippy::indexing_slicing, + clippy::shadow_unrelated, + reason = "test assertions on a JSON patch" + )] + fn the_status_column_is_written_with_the_condition_and_backfilled_once() { + let down = readiness::Verdict { + reason: readiness::Reason::NoEndpointsReady, + message: "0 ready endpoints".to_owned(), + }; + // The condition already says so, but STATE was never written: backfill it. + let unshown = provider_with_status(&serde_json::json!({ + "conditions": [ready_condition_json("False", "NoEndpointsReady")] + })); + let (_, patch) = ready_condition_patch(&unshown, &down).expect("state is backfilled"); + assert_eq!(patch["status"]["state"], "NotReady"); + assert_eq!( + patch["status"]["conditions"][0]["lastTransitionTime"], "2026-10-03T00:00:00Z", + "the held condition rides along unchanged, so apply does not drop it" + ); + // Both already current: nothing to write. + let shown = provider_with_status(&serde_json::json!({ + "conditions": [ready_condition_json("False", "NoEndpointsReady")], + "state": "NotReady" + })); + assert!(ready_condition_patch(&shown, &down).is_none()); + // A transition writes both. + let up = readiness::Verdict { + reason: readiness::Reason::Ready, + message: "1 ready endpoints".to_owned(), + }; + let (_, patch) = ready_condition_patch(&shown, &up).expect("a transition is written"); + assert_eq!(patch["status"]["state"], "Ready"); + assert_eq!(patch["status"]["conditions"][0]["status"], "True"); + } + fn network_with_modes(spec: &serde_json::Value) -> GridNetwork { serde_json::from_value(serde_json::json!({ "apiVersion": "grid.praxis.fast/v1alpha1", @@ -6610,6 +7029,7 @@ mod tests { network: "net".to_owned(), local_site: "site".to_owned(), candidates: Vec::new(), + excluded: Vec::new(), selection_policy: None, generated_at: Some("2026-07-29T01:00:00Z".to_owned()), }, diff --git a/operator/src/controller/inference_provider.rs b/operator/src/controller/inference_provider.rs index 9c859c852..cc9a86aae 100644 --- a/operator/src/controller/inference_provider.rs +++ b/operator/src/controller/inference_provider.rs @@ -646,7 +646,6 @@ pub(crate) fn sites_matching_selector(provider: &InferenceProvider, sites: &[Gri names } - /// The sites hosting `provider`: this site alone when its selector is empty and this site is in /// the provider's network, as the routing overlay attributes it, else every site the selector matches. fn hosting_sites(provider: &InferenceProvider, sites: &[GridSite], local_site: Option<&str>) -> Vec { @@ -699,6 +698,8 @@ async fn update_status( ModelDiscoveryConfig::OpenAiModels(openai) => openai.effective_url(&provider.spec.endpoint), }); let status = InferenceProviderStatus { + conditions: Vec::new(), + state: None, matching_sites, model_discovery_error: None, model_discovery_url, @@ -741,6 +742,8 @@ mod tests { #[test] fn reconciler_status_matches_only_reconciler_fields() { let baseline = InferenceProviderStatus { + conditions: Vec::new(), + state: None, matching_sites: vec!["site-a".to_owned()], model_discovery_error: None, model_discovery_url: None, @@ -769,6 +772,23 @@ mod tests { assert!(!baseline.matches_reconciler_status(&changed_discovery_url)); } + #[test] + fn a_conditions_change_is_not_the_reconcilers_to_write() { + let baseline = InferenceProviderStatus::default(); + let changed_conditions = InferenceProviderStatus { + conditions: vec![crate::crd::inference_provider::Condition { + type_: "Ready".to_owned(), + status: "False".to_owned(), + ..Default::default() + }], + ..baseline.clone() + }; + assert!( + baseline.matches_reconciler_status(&changed_conditions), + "the signals loop owns conditions" + ); + } + // ----------------------------------------------------------------------- // Test utilities // ----------------------------------------------------------------------- @@ -2667,6 +2687,7 @@ mod tests { ) -> InferenceProviderSpec { InferenceProviderSpec { capacity_weight: None, + max_running: None, grid_network_ref: "net".to_owned(), access_policy: crate::crd::auth::AccessPolicy::default(), auth: None, diff --git a/operator/src/crd/inference_provider.rs b/operator/src/crd/inference_provider.rs index e4ae8e907..e98e56108 100644 --- a/operator/src/crd/inference_provider.rs +++ b/operator/src/crd/inference_provider.rs @@ -29,7 +29,10 @@ use super::{ status = "InferenceProviderStatus", namespaced = false, printcolumn = r#"{"name":"Provider","type":"string","jsonPath":".spec.providerKind"}"#, - printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase"}"# + printcolumn = r#"{"name":"Status","type":"string","jsonPath":".status.state"}"#, + printcolumn = r#"{"name":"Age","type":"date","jsonPath":".metadata.creationTimestamp"}"#, + printcolumn = r#"{"name":"Reason","type":"string","jsonPath":".status.conditions[?(@.type==\"Ready\")].reason","priority":1}"#, + printcolumn = r#"{"name":"Phase","type":"string","jsonPath":".status.phase","priority":1}"# )] #[serde(rename_all = "camelCase")] pub struct InferenceProviderSpec { @@ -62,6 +65,15 @@ pub struct InferenceProviderSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub capacity_weight: Option, + /// Most requests one endpoint runs at once: vLLM max-num-seqs. + /// + /// The site's operator multiplies it by the EPP's fresh ready endpoints and publishes + /// the product as `grid_provider_capacity_requests`, the capacity gateways weigh the + /// provider's load against. When absent, or no endpoint is fresh, capacity is unknown. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_running: Option, + /// Cost information. pub cost: Option, @@ -443,6 +455,13 @@ pub struct MetricSignalNames { /// Metric name for a health gauge (any positive value = healthy). #[serde(skip_serializing_if = "Option::is_none")] pub healthy: Option, + + /// Metric name counting the pool's ready endpoints, read for the `Ready` condition. + /// + /// Defaults to `llm_d_epp_ready_endpoints`, then `inference_pool_ready_pods`. + /// Filtered by `poolName` when set. Zero marks the provider not ready. + #[serde(skip_serializing_if = "Option::is_none")] + pub ready_endpoints: Option, } /// Returns the default metrics scrape path. @@ -589,6 +608,20 @@ fn default_models_path() -> String { #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct InferenceProviderStatus { + /// Observed conditions. `Ready` says whether the provider can currently serve a request. + /// + /// Written by the operator's signals loop, never by provider reconciliation. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + #[schemars(extend("x-kubernetes-list-type" = "map", "x-kubernetes-list-map-keys" = ["type"]))] + pub conditions: Vec, + + /// `Ready`, `NotReady`, or `Unknown`: the `Ready` condition's status, for display. + /// + /// Written with the condition, never by provider reconciliation. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(extend("enum" = ["Ready", "NotReady", "Unknown"]))] + pub state: Option, + /// Sites matched by the site selector. #[serde(default)] pub matching_sites: Vec, @@ -638,6 +671,28 @@ impl InferenceProviderStatus { } } +/// One observed condition, shaped like `metav1.Condition`. +#[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Condition { + /// Condition type, such as `Ready`. + #[serde(rename = "type")] + pub type_: String, + /// `True`, `False`, or `Unknown`. + pub status: String, + /// CamelCase reason for the status. + pub reason: String, + /// Human-readable detail. + #[serde(default)] + pub message: String, + /// When `status` last changed, RFC 3339. + #[schemars(extend("format" = "date-time"))] + pub last_transition_time: String, + /// The `metadata.generation` this was computed against. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observed_generation: Option, +} + /// Lifecycle phase of a provider resource. #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] pub enum ProviderPhase { diff --git a/operator/src/latency.rs b/operator/src/latency.rs new file mode 100644 index 000000000..de6376674 --- /dev/null +++ b/operator/src/latency.rs @@ -0,0 +1,472 @@ +//! A provider's recent request latency, from deltas of its EPP's cumulative histograms. +//! +//! A value with too few completed requests in the window is left unpublished, so it reads as unknown. + +use std::{ + collections::{BTreeMap, VecDeque}, + time::{Duration, Instant}, +}; + +use crate::signals::Observation; + +/// How far back the published latency looks. +pub(crate) const WINDOW: Duration = Duration::from_secs(30); + +/// Completed requests a value needs in the window before it is published. +pub(crate) const MIN_SAMPLES: f64 = 20.0; + +/// Time to first token at the median, streaming requests only, seconds. +pub const TTFT_P50_SIGNAL: &str = "grid_provider_ttft_p50_seconds"; +/// Time to first token at the 90th percentile, streaming requests only, seconds. +pub const TTFT_P90_SIGNAL: &str = "grid_provider_ttft_p90_seconds"; +/// Mean time per output token, streaming requests only, seconds. +pub const TPOT_SIGNAL: &str = "grid_provider_tpot_seconds"; +/// Estimated prefill seconds per uncached input token. +pub const PREFILL_SIGNAL: &str = "grid_provider_prefill_seconds_per_token"; +/// Failed requests over all requests. +pub const ERROR_RATIO_SIGNAL: &str = "grid_provider_error_ratio"; + +/// EPP time to first token, labeled `streaming`. +const TTFT: &str = "llm_d_epp_request_ttft_seconds"; +/// EPP time per output token, streaming requests. +const TPOT: &str = "llm_d_epp_request_streaming_tpot_seconds"; +/// EPP input tokens per request. +const INPUT_TOKENS: &str = "llm_d_epp_request_input_tokens"; +/// EPP cached prompt tokens per request, when the model server reports them. +const CACHED_TOKENS: &str = "llm_d_epp_request_cached_tokens"; +/// EPP flow-control wait, labeled `inference_pool`, present only with flow control on. +const QUEUE_WAIT: &str = "llm_d_epp_flow_control_request_queue_duration_seconds"; +/// EPP requests processed. +const REQUESTS: &str = "llm_d_epp_request_total"; +/// EPP requests that failed. +const ERRORS: &str = "llm_d_epp_request_error_total"; + +/// A histogram's `_sum` and `_count`. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +struct SumCount { + /// Summed observations. + sum: f64, + /// Observations. + count: f64, +} + +impl SumCount { + /// `self - earlier`, `None` when a counter went backwards, as after an EPP restart. + fn since(self, earlier: Self) -> Option { + (self.sum >= earlier.sum && self.count >= earlier.count).then_some(Self { + sum: self.sum - earlier.sum, + count: self.count - earlier.count, + }) + } + + /// The mean observation, `None` with none. + fn mean(self) -> Option { + (self.count > 0.0).then(|| self.sum / self.count) + } +} + +/// The EPP's counters summed over models, fairness and priority: cumulative at one scrape, +/// or the difference across a window. +#[derive(Clone, Debug, Default, PartialEq)] +pub(crate) struct Snapshot { + /// Streaming TTFT bucket counts by upper bound, in bound order. + ttft_buckets: Vec<(f64, f64)>, + /// Streaming TTFT. + ttft: SumCount, + /// Streaming time per output token. + tpot: SumCount, + /// Input tokens per request. + input: SumCount, + /// Cached tokens, absent until a response reports them. + cached: Option, + /// Flow-control wait for the pool, absent without flow control. + queue_wait: Option, + /// Requests processed. + requests: f64, + /// Requests failed. + errors: f64, +} + +/// The sum of `metric` over the observations `keep` accepts, `None` when none match. +fn summed(observations: &[Observation], metric: &str, keep: impl Fn(&Observation) -> bool) -> Option { + observations + .iter() + .filter(|o| o.metric == metric && keep(o)) + .map(|o| o.value) + .reduce(|a, b| a + b) +} + +/// The `_sum` and `_count` of histogram `name` over the observations `keep` accepts. +fn sum_count(observations: &[Observation], name: &str, keep: impl Fn(&Observation) -> bool + Copy) -> Option { + Some(SumCount { + sum: summed(observations, &format!("{name}_sum"), keep)?, + count: summed(observations, &format!("{name}_count"), keep)?, + }) +} + +/// Whether a TTFT series counts streaming requests. +fn streaming(o: &Observation) -> bool { + o.labels.get("streaming").is_some_and(|value| value == "true") +} + +/// Streaming TTFT bucket counts by upper bound, summed across series. +fn ttft_buckets(observations: &[Observation]) -> Vec<(f64, f64)> { + let mut buckets: BTreeMap = BTreeMap::new(); + let bucket = format!("{TTFT}_bucket"); + for o in observations.iter().filter(|o| o.metric == bucket && streaming(o)) { + if let Some(bound) = o.labels.get("le").and_then(|le| le.parse::().ok()) { + // Bounds are positive, so their bit patterns sort as the values do. + buckets.entry(bound.to_bits()).or_insert((bound, 0.0)).1 += o.value; + } + } + buckets.into_values().collect() +} + +impl Snapshot { + /// The cumulative counters in one scrape, flow-control wait read for `pool`. + pub(crate) fn of(observations: &[Observation], pool: Option<&str>) -> Self { + let any = |_: &Observation| true; + let in_pool = + |o: &Observation| pool.is_none_or(|pool| o.labels.get("inference_pool").is_none_or(|p| p == pool)); + Self { + ttft_buckets: ttft_buckets(observations), + ttft: sum_count(observations, TTFT, streaming).unwrap_or_default(), + tpot: sum_count(observations, TPOT, any).unwrap_or_default(), + input: sum_count(observations, INPUT_TOKENS, any).unwrap_or_default(), + cached: sum_count(observations, CACHED_TOKENS, any), + queue_wait: sum_count(observations, QUEUE_WAIT, in_pool), + requests: summed(observations, REQUESTS, any).unwrap_or(0.0), + errors: summed(observations, ERRORS, any).unwrap_or(0.0), + } + } + + /// Whether the engine answered since `earlier`: a first token, a completion, or a usage report. + pub(crate) fn produced_since(&self, earlier: &Self) -> bool { + self.ttft.count > earlier.ttft.count + || self.tpot.count > earlier.tpot.count + || self.input.count > earlier.input.count + } + + /// `self - earlier`, `None` when any counter went backwards. + fn since(&self, earlier: &Self) -> Option { + let optional = |now: Option, then: Option| match (now, then) { + (Some(now), Some(then)) => now.since(then).map(Some), + (Some(now), None) => Some(Some(now)), + (None, _) => Some(None), + }; + (self.requests >= earlier.requests && self.errors >= earlier.errors).then_some(())?; + Some(Self { + ttft_buckets: self.bucket_deltas(earlier)?, + ttft: self.ttft.since(earlier.ttft)?, + tpot: self.tpot.since(earlier.tpot)?, + input: self.input.since(earlier.input)?, + cached: optional(self.cached, earlier.cached)?, + queue_wait: optional(self.queue_wait, earlier.queue_wait)?, + requests: self.requests - earlier.requests, + errors: self.errors - earlier.errors, + }) + } + + /// Each TTFT bucket's count less `earlier`'s, `None` when one went backwards. + fn bucket_deltas(&self, earlier: &Self) -> Option> { + self.ttft_buckets + .iter() + .map(|(bound, count)| { + let then = earlier + .ttft_buckets + .iter() + .find(|(b, _)| b.to_bits() == bound.to_bits()) + .map_or(0.0, |(_, earlier_count)| *earlier_count); + (*count >= then).then_some((*bound, count - then)) + }) + .collect() + } + + /// The series this window has enough samples to publish. + fn observations(&self) -> Vec { + let enough = |count: f64| count >= MIN_SAMPLES; + let mut published = Vec::new(); + if enough(self.ttft.count) { + published.extend(quantile(&self.ttft_buckets, 0.5).map(|value| sample(TTFT_P50_SIGNAL, value))); + published.extend(quantile(&self.ttft_buckets, 0.9).map(|value| sample(TTFT_P90_SIGNAL, value))); + } + if enough(self.tpot.count) { + published.extend(self.tpot.mean().map(|value| sample(TPOT_SIGNAL, value))); + } + if enough(self.ttft.count) && enough(self.input.count) { + published.extend( + self.prefill_seconds_per_token() + .map(|value| sample(PREFILL_SIGNAL, value)), + ); + } + if enough(self.requests) { + published.push(sample( + ERROR_RATIO_SIGNAL, + (self.errors / self.requests).clamp(0.0, 1.0), + )); + } + published + } + + /// Mean TTFT less mean flow-control wait, over mean uncached input tokens. + /// + /// Token means are per usage report. An engine reporting usage on every chunk weights + /// long outputs more, but each report carries the whole prompt, so the mean stays a + /// prompt length. Sensitive to the workload mix: TTFT counts streaming requests only, + /// while token counts cover every request. + fn prefill_seconds_per_token(&self) -> Option { + let wait = self.queue_wait.and_then(SumCount::mean).unwrap_or(0.0); + let cached = self.cached.map_or(0.0, |cached| cached.sum / self.input.count); + let uncached = self.input.mean()? - cached; + let prefill = self.ttft.mean()? - wait; + (uncached >= 1.0 && prefill > 0.0).then(|| prefill / uncached) + } +} + +/// One provider's recent cumulative snapshots. +#[derive(Clone, Debug, Default)] +pub(crate) struct History { + /// Snapshots by scrape time, oldest first, one at or before the window's start. + snapshots: VecDeque<(Instant, Snapshot)>, +} + +impl History { + /// The latest recorded scrape. + pub(crate) fn latest(&self) -> Option<&Snapshot> { + self.snapshots.back().map(|(_, snapshot)| snapshot) + } + + /// Record this scrape and return the latency series to publish for it. + pub(crate) fn record(&mut self, snapshot: Snapshot, now: Instant) -> Vec { + while self + .snapshots + .get(1) + .is_some_and(|(at, _)| now.saturating_duration_since(*at) >= WINDOW) + { + self.snapshots.pop_front(); + } + let published = self + .snapshots + .front() + .and_then(|(_, oldest)| snapshot.since(oldest)) + .map_or_else(Vec::new, |window| window.observations()); + if self + .snapshots + .back() + .is_some_and(|(_, last)| snapshot.since(last).is_none()) + { + // A counter reset: start over from this scrape. + self.snapshots.clear(); + } + self.snapshots.push_back((now, snapshot)); + published + } +} + +/// The `q` quantile of cumulative bucket counts by upper bound, interpolated within its bucket. +fn quantile(buckets: &[(f64, f64)], q: f64) -> Option { + let total = buckets.last()?.1; + if total <= 0.0 { + return None; + } + let rank = q * total; + let mut lower = (0.0, 0.0); + for (bound, count) in buckets { + if *count >= rank { + if bound.is_infinite() { + return Some(lower.0); + } + let within = count - lower.1; + let fraction = if within > 0.0 { (rank - lower.1) / within } else { 1.0 }; + return Some(lower.0 + (bound - lower.0) * fraction); + } + lower = (*bound, *count); + } + Some(lower.0) +} + +/// An unlabeled sample of `metric`. +fn sample(metric: &str, value: f64) -> Observation { + Observation { + metric: metric.to_owned(), + labels: BTreeMap::new(), + value, + timestamp_ms: None, + } +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::float_arithmetic, + clippy::arithmetic_side_effects, + clippy::indexing_slicing, + clippy::too_many_arguments, + reason = "tests" +)] +mod tests { + use super::*; + + impl History { + /// Record `observations` as one scrape, as the readiness store does. + fn record_observations( + &mut self, + observations: &[Observation], + pool: Option<&str>, + now: Instant, + ) -> Vec { + self.record(Snapshot::of(observations, pool), now) + } + } + + fn obs(metric: &str, labels: &[(&str, &str)], value: f64) -> Observation { + Observation { + metric: metric.to_owned(), + labels: labels.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned())).collect(), + value, + timestamp_ms: None, + } + } + + /// An EPP exposition after `requests` streaming requests, each with TTFT `ttft`, TPOT + /// `tpot`, `input` tokens of which `cached` were cached, and `errors` failures. + fn scrape(requests: f64, ttft: f64, tpot: f64, input: f64, cached: f64, errors: f64) -> Vec { + let streaming = [("streaming", "true"), ("model_name", "m")]; + let mut out = vec![]; + for bound in [0.1, 0.2, 0.4, 0.8, 1.6, f64::INFINITY] { + let le = if bound.is_infinite() { + "+Inf".to_owned() + } else { + bound.to_string() + }; + let count = if ttft <= bound { requests } else { 0.0 }; + out.push(obs( + &format!("{TTFT}_bucket"), + &[streaming[0], streaming[1], ("le", &le)], + count, + )); + } + out.push(obs(&format!("{TTFT}_sum"), &streaming, ttft * requests)); + out.push(obs(&format!("{TTFT}_count"), &streaming, requests)); + // A non-streaming series must not count. + out.push(obs(&format!("{TTFT}_sum"), &[("streaming", "false")], 999.0)); + out.push(obs(&format!("{TTFT}_count"), &[("streaming", "false")], 1.0)); + out.push(obs(&format!("{TPOT}_sum"), &[], tpot * requests)); + out.push(obs(&format!("{TPOT}_count"), &[], requests)); + out.push(obs(&format!("{INPUT_TOKENS}_sum"), &[], input * requests)); + out.push(obs(&format!("{INPUT_TOKENS}_count"), &[], requests)); + out.push(obs(&format!("{CACHED_TOKENS}_sum"), &[], cached * requests)); + out.push(obs(&format!("{CACHED_TOKENS}_count"), &[], requests)); + out.push(obs(REQUESTS, &[], requests)); + out.push(obs(ERRORS, &[("error_code", "500")], errors)); + out + } + + fn value(published: &[Observation], metric: &str) -> Option { + published.iter().find(|o| o.metric == metric).map(|o| o.value) + } + + #[test] + fn a_window_publishes_latency_from_the_requests_inside_it() { + let mut history = History::default(); + let start = Instant::now(); + let first = history.record_observations(&scrape(100.0, 0.3, 0.02, 1000.0, 200.0, 0.0), None, start); + assert!(first.is_empty(), "a first scrape has no window"); + let later = scrape(200.0, 0.3, 0.02, 1000.0, 200.0, 5.0); + let published = history.record_observations(&later, None, start + Duration::from_secs(10)); + let p50 = value(&published, TTFT_P50_SIGNAL).unwrap(); + assert!( + (0.2..=0.4).contains(&p50), + "every request in the 0.2..0.4 bucket: {p50}" + ); + assert!(value(&published, TTFT_P90_SIGNAL).unwrap() >= p50); + assert!((value(&published, TPOT_SIGNAL).unwrap() - 0.02).abs() < 1e-9); + let prefill = value(&published, PREFILL_SIGNAL).unwrap(); + assert!( + (prefill - 0.3 / 800.0).abs() < 1e-9, + "0.3s over 800 uncached tokens: {prefill}" + ); + assert!((value(&published, ERROR_RATIO_SIGNAL).unwrap() - 0.05).abs() < 1e-9); + } + + #[test] + fn too_few_requests_publish_nothing() { + let mut history = History::default(); + let start = Instant::now(); + history.record_observations(&scrape(100.0, 0.3, 0.02, 1000.0, 0.0, 0.0), None, start); + let published = history.record_observations( + &scrape(105.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(10), + ); + assert!(published.is_empty(), "5 requests is too few: {published:?}"); + } + + #[test] + fn a_counter_reset_publishes_nothing_and_starts_over() { + let mut history = History::default(); + let start = Instant::now(); + history.record_observations(&scrape(500.0, 0.3, 0.02, 1000.0, 0.0, 0.0), None, start); + let reset = history.record_observations( + &scrape(50.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(5), + ); + assert!(reset.is_empty(), "an EPP restart is not negative latency"); + let after = history.record_observations( + &scrape(100.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(10), + ); + assert!(value(&after, TPOT_SIGNAL).is_some(), "the window restarts at the reset"); + } + + #[test] + fn the_window_drops_snapshots_older_than_it() { + let mut history = History::default(); + let start = Instant::now(); + for (seconds, requests) in [(0, 0.0), (20, 100.0), (40, 100.0)] { + history.record_observations( + &scrape(requests, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(seconds), + ); + } + let idle = history.record_observations( + &scrape(100.0, 0.3, 0.02, 1000.0, 0.0, 0.0), + None, + start + Duration::from_secs(60), + ); + assert!(idle.is_empty(), "nothing completed in the last 30s: {idle:?}"); + } + + #[test] + fn queue_wait_comes_off_ttft_for_prefill() { + let window = Snapshot { + ttft: SumCount { + sum: 50.0, + count: 100.0, + }, + input: SumCount { + sum: 100_000.0, + count: 100.0, + }, + queue_wait: Some(SumCount { + sum: 20.0, + count: 100.0, + }), + requests: 100.0, + ..Snapshot::default() + }; + let prefill = window.prefill_seconds_per_token().unwrap(); + assert!((prefill - 0.3 / 1000.0).abs() < 1e-12, "(0.5 - 0.2) / 1000: {prefill}"); + } + + #[test] + fn quantiles_interpolate_within_a_bucket() { + let buckets = [(1.0, 0.0), (2.0, 10.0), (f64::INFINITY, 10.0)]; + assert!((quantile(&buckets, 0.5).unwrap() - 1.5).abs() < 1e-9); + assert_eq!(quantile(&[(1.0, 0.0), (f64::INFINITY, 0.0)], 0.5), None); + } +} diff --git a/operator/src/lib.rs b/operator/src/lib.rs index b38bf14af..bfc6b0bed 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -31,6 +31,8 @@ pub mod crd; pub mod enroll; /// Operator error types. pub mod error; +/// Provider readiness resolved from scraped metrics. +pub mod latency; /// Prometheus metrics for gateway probe and phase-transition observability. pub mod metrics; /// Pure Prometheus text-format parser for inference backend metrics. @@ -41,6 +43,7 @@ pub mod metrics_scraper; pub mod metrics_tls; /// Short-lived tokens for the metrics scraper ServiceAccount. pub(crate) mod metrics_token; +pub mod readiness; /// Kubernetes resource builders. pub mod resources; pub mod served_models; diff --git a/operator/src/main.rs b/operator/src/main.rs index 6ac0a2118..36ca891aa 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -178,8 +178,11 @@ async fn main() { .with_declared_trust(declared_trust) .with_rotation(rotation_running) .with_site_name(config.swim.site_name.clone()) + .with_scrape_interval(config.signals.scrape_interval()) .hold_membership(), ); + // This site's providers first, then what its peers publish, on /metrics. + operator::metrics::register_provider_signals(vec![ctx.signals(), ctx.peers()]); // Controllers run now. Only SWIM and what dials peers wait on the advertise address. let (swim_tx, swim_rx) = tokio::sync::watch::channel(SwimStage::Starting); diff --git a/operator/src/metrics.rs b/operator/src/metrics.rs index 999cbad8f..1ab36aee0 100644 --- a/operator/src/metrics.rs +++ b/operator/src/metrics.rs @@ -39,6 +39,12 @@ static REGISTRY: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_RETRIES.clone())) .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(PEER_SIGNALS_REFUSED.clone())) + .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(PROVIDER_SCRAPES.clone())) + .unwrap_or_else(|_| std::process::abort()); + r.register(Box::new(PROVIDER_LAST_SCRAPE_SUCCESS.clone())) + .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_DURATION.clone())) .unwrap_or_else(|_| std::process::abort()); r.register(Box::new(PEER_POLL_SLOW.clone())) @@ -107,7 +113,6 @@ static SWIM_PENDING_DROPS: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()) }); - // --------------------------------------------------------------------------- // Peer polling // @@ -125,6 +130,37 @@ static PEER_POLL_TOTAL: LazyLock = LazyLock::new(|| { .unwrap_or_else(|_| std::process::abort()) }); +/// Provider metrics scrapes by provider and result: `success`, `no_series`, or a failure class +/// (`timeout`, `unauthorized`, `tls`, `dns`, `connect`, `http`, `body_cap`, `parse`, `config`). +static PROVIDER_SCRAPES: LazyLock = LazyLock::new(|| { + IntCounterVec::new( + Opts::new("grid_provider_scrape_total", "Provider metrics scrapes by result"), + &["grid_provider", "result"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + +/// When each provider's metrics last scraped with its ready-endpoint series, Unix seconds. +static PROVIDER_LAST_SCRAPE_SUCCESS: LazyLock = LazyLock::new(|| { + IntGaugeVec::new( + Opts::new( + "grid_provider_last_scrape_success_timestamp_seconds", + "Unix time of the provider's last scrape with its ready-endpoint series", + ), + &["grid_provider"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + +/// Peer observations refused at ingest, by peer and reason. +static PEER_SIGNALS_REFUSED: LazyLock = LazyLock::new(|| { + IntCounterVec::new( + Opts::new("grid_peer_signals_refused_total", "Peer observations refused at ingest"), + &["peer", "reason"], + ) + .unwrap_or_else(|_| std::process::abort()) +}); + /// Retried attempts by peer and the outcome that prompted the retry. /// /// Separate from the poll counter because a poll that succeeded on its third @@ -427,6 +463,44 @@ pub(crate) fn record_peer_poll(peer: &str, outcome: &str, duration: Duration, by } } +/// Count one scrape of `provider` with `result`. +pub(crate) fn record_provider_scrape(provider: &str, result: &str) { + PROVIDER_SCRAPES.with_label_values(&[provider, result]).inc(); +} + +/// Record that `provider` last scraped with its ready-endpoint series at `at`. +pub(crate) fn set_provider_last_scrape_success(provider: &str, at: std::time::SystemTime) { + let secs = at.duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs(); + PROVIDER_LAST_SCRAPE_SUCCESS + .with_label_values(&[provider]) + .set(secs.try_into().unwrap_or(i64::MAX)); +} + +/// Drop `provider`'s scrape series once it is gone, so they do not outlive it. +pub(crate) fn forget_provider_scrapes(provider: &str) { + let _absent = PROVIDER_LAST_SCRAPE_SUCCESS.remove_label_values(&[provider]); + for result in [ + "success", + "no_series", + "timeout", + "unauthorized", + "tls", + "dns", + "connect", + "http", + "body_cap", + "parse", + "config", + ] { + let _absent_result = PROVIDER_SCRAPES.remove_label_values(&[provider, result]); + } +} + +/// Count a peer observation this hub refused: `name`, `provider`, `value`, or `provider_cap`. +pub(crate) fn record_peer_signal_refused(peer: &str, reason: &str) { + PEER_SIGNALS_REFUSED.with_label_values(&[peer, reason]).inc(); +} + /// Record an attempt that failed and will be tried again. pub(crate) fn record_peer_retry(peer: &str, reason: &str) { PEER_POLL_RETRIES.with_label_values(&[peer, reason]).inc(); @@ -476,6 +550,116 @@ pub fn record_site_identity_renewal(result: &str) { SITE_IDENTITY_RENEWALS.with_label_values(&[result]).inc(); } +/// Provider series exported on `/metrics`, as each site's operator resolves them. +const PROVIDER_SIGNALS: [(&str, &str); 9] = [ + ("grid_provider_ready", "1 when the provider can serve, 0 when not."), + ( + "grid_provider_in_flight_requests", + "Requests the provider holds, running, engine-queued, and held by flow control.", + ), + ( + "grid_provider_capacity_requests", + "maxRunning per endpoint times fresh ready endpoints.", + ), + ("grid_provider_saturation_ratio", "Requests held over capacity."), + ( + "grid_provider_ttft_p50_seconds", + "Median streaming time to first token over the last 30s.", + ), + ( + "grid_provider_ttft_p90_seconds", + "90th percentile streaming time to first token over the last 30s.", + ), + ( + "grid_provider_tpot_seconds", + "Mean streaming time per output token over the last 30s.", + ), + ( + "grid_provider_prefill_seconds_per_token", + "Estimated prefill seconds per uncached input token over the last 30s. Moves with the workload mix.", + ), + ( + "grid_provider_error_ratio", + "Failed requests over all requests in the last 30s.", + ), +]; + +/// Labels on each exported provider series: both bounded by the grid's sites and providers. +const PROVIDER_LABELS: [&str; 2] = ["grid_site", "grid_provider"]; + +/// Exports the provider series held in `stores` at each scrape, this site's own and those +/// polled from peers. A series this site does not hold is absent, not 0. +struct ProviderSignals { + /// The local and peer signal stores. + stores: Vec, + /// One gauge per exported series, paired with its metric name so a failed constructor + /// drops only its own series. + gauges: Vec<(&'static str, prometheus::GaugeVec)>, + /// Serializes a scrape's reset and refill of the shared gauges. + collecting: std::sync::Mutex<()>, +} + +impl prometheus::core::Collector for ProviderSignals { + fn desc(&self) -> Vec<&prometheus::core::Desc> { + self.gauges.iter().flat_map(|(_, gauge)| gauge.desc()).collect() + } + + fn collect(&self) -> Vec { + let _collecting = self + .collecting + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for (_, gauge) in &self.gauges { + gauge.reset(); + } + let mut seen = std::collections::BTreeSet::new(); + for sample in self.stores.iter().flat_map(crate::signals::SignalStore::current) { + let (Some(site), Some(provider)) = (sample.labels.get("grid_site"), sample.labels.get("grid_provider")) + else { + continue; + }; + let Some((_, gauge)) = self.gauges.iter().find(|(name, _)| *name == sample.metric) else { + continue; + }; + // The first store holding a series wins: this site's own before a peer's. + if seen.insert((sample.metric.clone(), site.clone(), provider.clone())) { + gauge + .with_label_values(&[site.as_str(), provider.as_str()]) + .set(sample.value); + } + } + self.gauges + .iter() + .flat_map(|(_, gauge)| prometheus::core::Collector::collect(gauge)) + .collect() + } +} + +/// Export the provider series held in `stores` on `/metrics`. Call once, with this site's +/// store first. +pub fn register_provider_signals(stores: Vec) { + if let Err(error) = REGISTRY.register(Box::new(provider_signals(stores))) { + tracing::warn!(%error, "metrics: provider signals already registered"); + } +} + +/// The collector over `stores`. +fn provider_signals(stores: Vec) -> ProviderSignals { + let gauges = PROVIDER_SIGNALS + .iter() + .filter_map(|(name, help)| { + prometheus::GaugeVec::new(Opts::new(*name, *help), &PROVIDER_LABELS) + .ok() + .map(|gauge| (*name, gauge)) + }) + .collect(); + ProviderSignals { + stores, + gauges, + collecting: std::sync::Mutex::new(()), + } +} + /// Gather all registered metrics for serialization. pub(crate) fn gather_metrics() -> Vec { expire_site_phases(); @@ -645,4 +829,135 @@ mod tests { "output should contain duration histogram" ); } + + #[test] + fn provider_scrapes_are_counted_by_result_and_forgotten_with_the_provider() { + let provider = "scrape-count-test"; + record_provider_scrape(provider, "success"); + record_provider_scrape(provider, "no_series"); + record_provider_scrape(provider, "no_series"); + set_provider_last_scrape_success(provider, std::time::UNIX_EPOCH + Duration::from_secs(42)); + assert_eq!(PROVIDER_SCRAPES.with_label_values(&[provider, "success"]).get(), 1); + assert_eq!(PROVIDER_SCRAPES.with_label_values(&[provider, "no_series"]).get(), 2); + assert_eq!(PROVIDER_LAST_SCRAPE_SUCCESS.with_label_values(&[provider]).get(), 42); + forget_provider_scrapes(provider); + assert!( + PROVIDER_SCRAPES.remove_label_values(&[provider, "success"]).is_err(), + "forgotten with the provider" + ); + assert!(PROVIDER_LAST_SCRAPE_SUCCESS.remove_label_values(&[provider]).is_err()); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one store across two scrapes")] + fn provider_signals_drop_a_series_the_store_no_longer_holds() { + use prometheus::core::Collector as _; + let ready = crate::signals::Observation { + metric: "grid_provider_ready".to_owned(), + labels: [("grid_site", "hq"), ("grid_provider", "pool")] + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(), + value: 1.0, + timestamp_ms: None, + }; + let store = crate::signals::SignalStore::new(); + let collector = provider_signals(vec![store.clone()]); + let count = |families: &[MetricFamily]| -> usize { + families + .iter() + .filter(|family| family.name() == "grid_provider_ready") + .map(|family| family.get_metric().len()) + .sum() + }; + store.refresh( + std::collections::BTreeMap::from([("pool".to_owned(), vec![ready])]), + Duration::from_secs(60), + ); + assert_eq!(count(&collector.collect()), 1, "a held series is exported"); + store.refresh( + std::collections::BTreeMap::from([("pool".to_owned(), Vec::new())]), + Duration::from_secs(60), + ); + assert_eq!( + count(&collector.collect()), + 0, + "a dropped series is not exported from the last scrape" + ); + assert_eq!( + collector.desc().len(), + PROVIDER_SIGNALS.len(), + "one desc per exported series" + ); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one scenario across both stores")] + fn provider_signals_export_this_site_and_its_peers_and_omit_what_is_not_held() { + use prometheus::core::Collector as _; + let held = |site: &str, metric: &str, value: f64| crate::signals::Observation { + metric: metric.to_owned(), + labels: [("grid_site", site), ("grid_provider", "pool")] + .iter() + .map(|(k, v)| ((*k).to_owned(), (*v).to_owned())) + .collect(), + value, + timestamp_ms: None, + }; + let local = crate::signals::SignalStore::new(); + local.refresh( + std::collections::BTreeMap::from([( + "pool".to_owned(), + vec![ + held("hq", "grid_provider_saturation_ratio", 0.5), + held("hq", "grid_provider_ttft_p50_seconds", 0.2), + held("hq", "llm_d_epp_average_queue_size", 3.0), + ], + )]), + Duration::from_secs(60), + ); + let peers = crate::signals::SignalStore::new(); + peers.refresh( + std::collections::BTreeMap::from([( + "retail".to_owned(), + vec![ + held("retail", "grid_provider_saturation_ratio", 0.9), + held("hq", "grid_provider_saturation_ratio", 7.0), + ], + )]), + Duration::from_secs(60), + ); + let families = provider_signals(vec![local, peers]).collect(); + let series = |name: &str| -> Vec<(String, f64)> { + families + .iter() + .filter(|family| family.name() == name) + .flat_map(|family| family.get_metric().iter()) + .map(|metric| { + let site = metric + .get_label() + .iter() + .find(|label| label.name() == "grid_site") + .map(|label| label.value().to_owned()) + .unwrap_or_default(); + (site, metric.get_gauge().value()) + }) + .collect() + }; + let mut saturation = series("grid_provider_saturation_ratio"); + saturation.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + saturation, + [("hq".to_owned(), 0.5), ("retail".to_owned(), 0.9)], + "this site's value wins" + ); + assert_eq!(series("grid_provider_ttft_p50_seconds"), [("hq".to_owned(), 0.2)]); + assert!(series("grid_provider_tpot_seconds").is_empty(), "not held, not 0"); + assert!( + families + .iter() + .all(|family| family.name() != "llm_d_epp_average_queue_size"), + "only provider series" + ); + } } diff --git a/operator/src/metrics_scraper.rs b/operator/src/metrics_scraper.rs index 8fb2e1488..2772e5066 100644 --- a/operator/src/metrics_scraper.rs +++ b/operator/src/metrics_scraper.rs @@ -93,6 +93,58 @@ pub enum MetricsScrapeError { /// A credential would have gone over plain HTTP without `allowPlaintext`. #[error("refusing to send a metrics credential over plain http: {0}")] PlaintextCredential(String), + /// The body passed the 1 MiB response limit. + #[error("metrics response body exceeds {0} byte limit")] + BodyTooLarge(usize), +} + +impl MetricsScrapeError { + /// The bounded class of this failure, for the provider's condition and scrape counters. + #[must_use] + pub(crate) fn class(&self) -> crate::readiness::ScrapeClass { + use crate::readiness::ScrapeClass; + match self { + Self::Timeout(_) => ScrapeClass::Timeout, + Self::NonOkStatus { status: 401 | 403, .. } => ScrapeClass::Unauthorized, + Self::NonOkStatus { .. } => ScrapeClass::Http, + Self::Transport(source) => transport_class(source.as_ref()), + Self::Encoding(_) => ScrapeClass::Parse, + Self::TlsMaterial(_) => ScrapeClass::Tls, + Self::BodyTooLarge(_) => ScrapeClass::BodyCap, + Self::InvalidUrl(_) | Self::HttpWithTls(_) | Self::Credential(_) | Self::PlaintextCredential(_) => { + ScrapeClass::Config + }, + } + } +} + +/// TLS when the chain holds a TLS error, DNS when resolution failed, timeout when the socket +/// timed out, else a connect failure. +fn transport_class(source: &(dyn std::error::Error + 'static)) -> crate::readiness::ScrapeClass { + use crate::readiness::ScrapeClass; + let mut cause = Some(source); + while let Some(error) = cause { + // io::Error's source skips the error it wraps, which is where a TLS failure sits. + let wrapped = error + .downcast_ref::() + .and_then(std::io::Error::get_ref) + .is_some_and(|inner| crate::resources::tls_backend::is_tls_error(inner)); + if wrapped || crate::resources::tls_backend::is_tls_error(error) { + return ScrapeClass::Tls; + } + let text = error.to_string(); + if text.contains("dns error") || text.contains("failed to lookup address") { + return ScrapeClass::Dns; + } + if error + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::TimedOut) + { + return ScrapeClass::Timeout; + } + cause = error.source(); + } + ScrapeClass::Connect } // --------------------------------------------------------------------------- @@ -299,9 +351,7 @@ async fn read_response( .await .map_err(|e| { if e.downcast_ref::().is_some() { - MetricsScrapeError::Transport( - format!("metrics response body exceeds {MAX_RESPONSE_BODY_BYTES} byte limit").into(), - ) + MetricsScrapeError::BodyTooLarge(MAX_RESPONSE_BODY_BYTES) } else { MetricsScrapeError::Transport(e) } @@ -327,6 +377,74 @@ mod tests { use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use super::*; + + #[test] + #[cfg(not(feature = "fips"))] + fn a_tls_failure_wrapped_in_an_io_error_classes_as_tls() { + let tls: Box = Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidData, + rustls::Error::General("bad certificate".to_owned()), + )); + assert_eq!( + MetricsScrapeError::Transport(tls).class(), + crate::readiness::ScrapeClass::Tls + ); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one table of failures and their classes")] + fn a_scrape_failure_has_a_bounded_class() { + use crate::readiness::ScrapeClass; + let status = |status| MetricsScrapeError::NonOkStatus { + status, + url: "https://epp:9090/metrics".to_owned(), + }; + let transport = |error: Box| MetricsScrapeError::Transport(error); + let cases = [ + ( + "timeout", + MetricsScrapeError::Timeout(Duration::from_secs(2)), + ScrapeClass::Timeout, + ), + ("401", status(401), ScrapeClass::Unauthorized), + ("403", status(403), ScrapeClass::Unauthorized), + ("500", status(500), ScrapeClass::Http), + ("404", status(404), ScrapeClass::Http), + ( + "dns", + transport("dns error: failed to lookup address information".into()), + ScrapeClass::Dns, + ), + ( + "refused", + transport(Box::new(std::io::Error::from(std::io::ErrorKind::ConnectionRefused))), + ScrapeClass::Connect, + ), + ( + "socket timeout", + transport(Box::new(std::io::Error::from(std::io::ErrorKind::TimedOut))), + ScrapeClass::Timeout, + ), + ( + "body cap", + MetricsScrapeError::BodyTooLarge(MAX_RESPONSE_BODY_BYTES), + ScrapeClass::BodyCap, + ), + ( + "tls material", + MetricsScrapeError::TlsMaterial("no key".to_owned()), + ScrapeClass::Tls, + ), + ( + "plaintext", + MetricsScrapeError::PlaintextCredential("http://epp".to_owned()), + ScrapeClass::Config, + ), + ]; + for (label, error, class) in cases { + assert_eq!(error.class(), class, "{label}"); + } + } use crate::resources::tls_backend::{MAX_CA_PEM_BYTES, MAX_CLIENT_CERT_PEM_BYTES, MAX_CLIENT_KEY_PEM_BYTES}; /// Start a local HTTP server on a random port and return the URL. diff --git a/operator/src/metrics_token.rs b/operator/src/metrics_token.rs index b4ffad245..1a9be1ff7 100644 --- a/operator/src/metrics_token.rs +++ b/operator/src/metrics_token.rs @@ -11,7 +11,7 @@ use std::{ }; use k8s_openapi::api::{ - authentication::v1::{BoundObjectReference, TokenRequest, TokenRequestSpec}, + authentication::v1::{TokenRequest, TokenRequestSpec}, core::v1::ServiceAccount, }; use kube::api::{Api, PostParams}; @@ -32,10 +32,6 @@ const RETRY_BACKOFF: Duration = Duration::from_secs(30); const SERVICE_ACCOUNT_ENV: &str = "GRID_METRICS_SCRAPER_SERVICE_ACCOUNT"; /// The namespace of [`SERVICE_ACCOUNT_ENV`]. const NAMESPACE_ENV: &str = "GRID_METRICS_SCRAPER_NAMESPACE"; -/// This pod's name and uid, from the downward API, to bind each token to the pod. -const POD_NAME_ENV: &str = "GRID_POD_NAME"; -/// See [`POD_NAME_ENV`]. -const POD_UID_ENV: &str = "GRID_POD_UID"; /// A minted token and its timeline. struct Cached { @@ -56,6 +52,8 @@ struct State { cached: Option, /// After a failed mint with no usable token, the earliest moment to try again. no_mint_before: Option, + /// The last mint failure logged, so a failure that repeats logs once. + failing: Option, } impl State { @@ -67,6 +65,9 @@ impl State { ) -> Result, MetricsScrapeError> { let error = match minted { Ok((token, lifetime)) => { + if self.failing.take().is_some() { + tracing::info!("metrics scraper token minted again"); + } *self = Self { cached: Some(Cached { token: token.clone(), @@ -75,19 +76,31 @@ impl State { retry_at: None, }), no_mint_before: None, + failing: None, }; return Ok(token); }, Err(error) => error, }; + self.note_failure(&error, now); if let Some(cached) = self.cached.as_mut().filter(|cached| now < cached.expires_at) { - tracing::warn!(%error, "metrics scraper token refresh failed; using the held token until it expires"); cached.retry_at = Some(now + RETRY_BACKOFF); return Ok(cached.token.clone()); } self.no_mint_before = Some(now + RETRY_BACKOFF); Err(error) } + + /// Log a mint failure at WARN once, until a different one or a success. + fn note_failure(&mut self, error: &MetricsScrapeError, now: Instant) { + let message = error.to_string(); + if self.failing.as_deref() == Some(message.as_str()) { + return; + } + let held = self.cached.as_ref().is_some_and(|cached| now < cached.expires_at); + tracing::warn!(%error, held, "metrics scraper token mint failed"); + self.failing = Some(message); + } } /// What a scrape does with the cached token at `now`. @@ -115,16 +128,20 @@ fn token_plan(state: &State, now: Instant) -> Plan { /// The current token, shared by every scrape. static CACHE: LazyLock> = LazyLock::new(|| tokio::sync::Mutex::new(State::default())); -/// This pod, when the downward API names it, as the object each token is bound to. -fn bound_pod() -> Option { - let name = std::env::var(POD_NAME_ENV).ok().filter(|v| !v.trim().is_empty())?; - let uid = std::env::var(POD_UID_ENV).ok().filter(|v| !v.trim().is_empty())?; - Some(BoundObjectReference { - api_version: Some("v1".to_owned()), - kind: Some("Pod".to_owned()), - name: Some(name), - uid: Some(uid), - }) +/// The `TokenRequest` for a scraper token. +/// +/// Unbound: the API server binds a token only to a pod running as the token's own +/// `ServiceAccount`, and the operator runs as another. +fn token_request() -> TokenRequest { + TokenRequest { + spec: TokenRequestSpec { + // The API server's own audience: an EPP's `TokenReview` checks no other. + audiences: Vec::new(), + expiration_seconds: i64::try_from(TOKEN_LIFETIME.as_secs()).ok(), + bound_object_ref: None, + }, + ..TokenRequest::default() + } } /// The configured scraper `ServiceAccount`, read once. @@ -147,15 +164,7 @@ async fn mint( name: &str, namespace: &str, ) -> Result<(Zeroizing, Duration), MetricsScrapeError> { - let request = TokenRequest { - spec: TokenRequestSpec { - // The API server's own audience: an EPP's `TokenReview` checks no other. - audiences: Vec::new(), - expiration_seconds: i64::try_from(TOKEN_LIFETIME.as_secs()).ok(), - bound_object_ref: bound_pod(), - }, - ..TokenRequest::default() - }; + let request = token_request(); let api = Api::::namespaced(client.clone(), namespace); let issued = tokio::time::timeout( MINT_TIMEOUT, @@ -224,6 +233,7 @@ mod tests { retry_at: retry_in.map(|secs| now + Duration::from_secs(secs)), }), no_mint_before: None, + failing: None, } } @@ -256,6 +266,7 @@ mod tests { let failed = State { cached: None, no_mint_before: Some(now + Duration::from_secs(30)), + failing: None, }; assert_eq!(token_plan(&failed, now), Plan::Fail, "backing off"); assert_eq!( @@ -265,6 +276,37 @@ mod tests { ); } + #[test] + fn a_token_is_unbound_for_the_api_servers_audience() { + let spec = token_request().spec; + assert!( + spec.bound_object_ref.is_none(), + "a pod running as another ServiceAccount cannot hold the binding" + ); + assert!(spec.audiences.is_empty(), "the API server's own audience"); + assert_eq!(spec.expiration_seconds, Some(600)); + } + + #[test] + fn a_repeated_mint_failure_logs_once_and_a_success_clears_it() { + let now = Instant::now(); + let mut state = State::default(); + let fail = || Err(MetricsScrapeError::Credential("refused".to_owned())); + assert_eq!(state.record(fail(), now).map(drop).ok(), None); + assert!( + state + .failing + .as_deref() + .is_some_and(|logged| logged.contains("refused")) + ); + let logged = state.failing.clone(); + assert_eq!(state.record(fail(), now).map(drop).ok(), None); + assert_eq!(state.failing, logged, "the same failure is not logged again"); + let token = state.record(Ok((Zeroizing::new("t".to_owned()), TOKEN_LIFETIME)), now); + assert_eq!(token.as_deref().map(String::as_str).ok(), Some("t")); + assert!(state.failing.is_none()); + } + #[test] fn the_api_servers_expiry_wins_over_the_requested_lifetime() { let now = UNIX_EPOCH + Duration::from_secs(1_000_000); diff --git a/operator/src/readiness.rs b/operator/src/readiness.rs new file mode 100644 index 000000000..a717c49cc --- /dev/null +++ b/operator/src/readiness.rs @@ -0,0 +1,1089 @@ +//! Whether each provider can serve a request now, resolved from its scraped metrics. +//! +//! The signals loop records each scrape here. The verdict feeds three readers: the +//! provider's `Ready` condition, the `grid_provider_ready` series peers poll, and +//! admission in the serving config. + +use std::{ + collections::HashMap, + sync::{Mutex, PoisonError}, + time::{Duration, Instant}, +}; + +use crate::{crd::inference_provider::Condition, signals::Observation}; + +/// Metric names that count a pool's ready endpoints, preferred first. +pub(crate) const DEFAULT_READY_ENDPOINTS: [&str; 2] = ["llm_d_epp_ready_endpoints", "inference_pool_ready_pods"]; + +/// Consecutive scrapes reading zero ready endpoints before a provider is not ready, +/// and reading some before it is ready again. +/// +/// One reading can catch a pod between its metrics going stale and a fresh one, and a +/// pool that just drained can read ready once before its first request lands. +pub(crate) const STREAK: u32 = 2; + +/// How recently the engine must have answered for a zero count to read as busy, not down. +/// +/// The EPP counts endpoints whose metrics are fresh, so a saturated engine whose /metrics +/// answers late reads as zero while it still serves. +pub(crate) const PROGRESS_WINDOW: Duration = Duration::from_secs(30); + +/// The series this site publishes per provider declaring `maxRunning`: that capacity. +pub const CAPACITY_SIGNAL: &str = "grid_provider_capacity_requests"; + +/// The series this site publishes per provider with both counts: requests held over capacity. +pub const SATURATION_SIGNAL: &str = "grid_provider_saturation_ratio"; + +/// The series this site publishes per provider: 1 when ready, 0 when not. +pub const READY_SIGNAL: &str = "grid_provider_ready"; + +/// The resolved signal for requests a provider has running and queued. +pub const IN_FLIGHT_SIGNAL: &str = "grid_provider_in_flight_requests"; + +/// The EPP's per-endpoint in-flight count, from its inflight-load-producer plugin. It +/// increments after flow control admits a request, once per scheduling profile's target, so +/// on a P/D pool a request counts on its prefill and its decode endpoint. +const EPP_IN_FLIGHT: &str = "llm_d_epp_inflight_requests"; + +/// Requests the EPP's flow control holds before scheduling, labeled by `inference_pool`. +const EPP_FLOW_CONTROL_QUEUE: &str = "llm_d_epp_flow_control_queue_size"; + +/// Pool averages that stand in when the EPP runs no inflight-load-producer. +const EPP_AVERAGES: [[&str; 2]; 2] = [ + [ + "llm_d_epp_average_running_requests", + "inference_pool_average_running_requests", + ], + ["llm_d_epp_average_queue_size", "inference_pool_average_queue_size"], +]; + +/// The `Ready` condition's status and reason for one provider. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Reason { + /// Scraped recently with at least one ready endpoint. + Ready, + /// The latest scrapes counted zero ready endpoints. + NoEndpointsReady, + /// The scrape answered, but without the pool's ready-endpoint series. + NoLivenessCheck, + /// No successful scrape within the staleness window, and no attempt failed. + MetricsStale, + /// No successful scrape within the window, and the latest attempt timed out. + ScrapeTimedOut, + /// No successful scrape within the window, and the latest attempt was refused (401 or 403). + ScrapeUnauthorized, + /// No successful scrape within the window, and the latest attempt failed TLS. + TlsHandshakeFailed, + /// No successful scrape within the window, and the latest attempt failed otherwise. + ScrapeFailed, + /// The provider itself is `Unavailable`. + ProviderUnavailable, + /// The provider declares no metrics, so readiness is unknown. + MetricsNotConfigured, + /// Attempted, with no scrape succeeding yet inside the grace window. + AwaitingFirstScrape, +} + +impl Reason { + /// The condition reason as written to status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Ready => "Ready", + Self::NoEndpointsReady => "NoEndpointsReady", + Self::NoLivenessCheck => "NoLivenessCheck", + Self::MetricsStale => "MetricsStale", + Self::ScrapeTimedOut => "ScrapeTimedOut", + Self::ScrapeUnauthorized => "ScrapeUnauthorized", + Self::TlsHandshakeFailed => "TLSHandshakeFailed", + Self::ScrapeFailed => "ScrapeFailed", + Self::ProviderUnavailable => "ProviderUnavailable", + Self::MetricsNotConfigured => "MetricsNotConfigured", + Self::AwaitingFirstScrape => "AwaitingFirstScrape", + } + } + + /// The condition status: `True`, `False`, or `Unknown`. + #[must_use] + pub const fn status(self) -> &'static str { + match self { + Self::Ready => "True", + Self::MetricsNotConfigured | Self::AwaitingFirstScrape => "Unknown", + Self::NoEndpointsReady + | Self::NoLivenessCheck + | Self::MetricsStale + | Self::ScrapeTimedOut + | Self::ScrapeUnauthorized + | Self::TlsHandshakeFailed + | Self::ScrapeFailed + | Self::ProviderUnavailable => "False", + } + } + + /// The one-word status shown in the STATUS column: `Ready`, `NotReady`, or `Unknown`. + #[must_use] + pub const fn display(self) -> &'static str { + match self { + Self::Ready => "Ready", + Self::MetricsNotConfigured | Self::AwaitingFirstScrape => "Unknown", + Self::NoEndpointsReady + | Self::NoLivenessCheck + | Self::MetricsStale + | Self::ScrapeTimedOut + | Self::ScrapeUnauthorized + | Self::TlsHandshakeFailed + | Self::ScrapeFailed + | Self::ProviderUnavailable => "NotReady", + } + } + + /// Whether the provider is known not to serve: unknown counts as ready. + #[must_use] + pub const fn excludes(self) -> bool { + !matches!( + self, + Self::Ready | Self::MetricsNotConfigured | Self::AwaitingFirstScrape + ) + } +} + +/// Why one scrape failed, bounded so it can label a metric. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ScrapeClass { + /// The request timed out. + Timeout, + /// The endpoint answered 401 or 403. + Unauthorized, + /// TLS failed: a handshake, a certificate, or the TLS material. + Tls, + /// The endpoint's name did not resolve. + Dns, + /// The connection failed. + Connect, + /// The endpoint answered with another non-2xx status. + Http, + /// The body passed the size limit. + BodyCap, + /// The body could not be read as text. + Parse, + /// The scrape could not be built: the URL, the credential, or plaintext refused. + Config, +} + +impl ScrapeClass { + /// The class as a metric label and message word. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Timeout => "timeout", + Self::Unauthorized => "unauthorized", + Self::Tls => "tls", + Self::Dns => "dns", + Self::Connect => "connect", + Self::Http => "http", + Self::BodyCap => "body_cap", + Self::Parse => "parse", + Self::Config => "config", + } + } + + /// The `Ready` reason when this was the latest failure and nothing succeeded since. + const fn reason(self) -> Reason { + match self { + Self::Timeout => Reason::ScrapeTimedOut, + Self::Unauthorized => Reason::ScrapeUnauthorized, + Self::Tls => Reason::TlsHandshakeFailed, + Self::Dns | Self::Connect | Self::Http | Self::BodyCap | Self::Parse | Self::Config => Reason::ScrapeFailed, + } + } +} + +/// What the signals loop last saw from one provider. +#[derive(Clone, Debug, Default)] +struct Probe { + /// When the first scrape was attempted, which starts the grace before any success. + first_attempt: Option, + /// When the last scrape succeeded. + last_good: Option, + /// Ready endpoints in that scrape, `None` when it exposed no count. + ready_endpoints: Option, + /// Signals from the latest success, until the signals loop publishes them once. + unpublished: Option>, + /// The latest attempt's failure, `None` when it succeeded. + failure: Option, + /// What the latest success lacked, when it exposed no ready-endpoint count. + missing: Option, + /// Consecutive successful scrapes that read zero ready endpoints. + zero_streak: u32, + /// Consecutive successful scrapes that read some ready endpoints. + ready_streak: u32, + /// Whether the count marks the provider not ready, with [`STREAK`] hysteresis both ways. + no_endpoints: bool, + /// When the EPP last recorded an engine answer for this provider. + last_progress: Option, + /// Whether the latest zero count came while the engine was answering. + busy: bool, + /// Recent EPP latency snapshots. + latency: crate::latency::History, +} + +/// One provider's verdict and the detail behind it. +#[derive(Clone, Debug, PartialEq)] +pub struct Verdict { + /// Status and reason. + pub reason: Reason, + /// Human detail for the condition message. + pub message: String, +} + +/// The latest scrape per provider, keyed by network and routing identity. +#[derive(Debug, Default)] +pub struct ReadinessStore(Mutex>); + +/// The store key for `identity` in `network`: identities are unique per network only. +pub(crate) fn key(network: &str, identity: &str) -> String { + format!("{network}/{identity}") +} + +impl ReadinessStore { + /// The probes, recovered if a panicking holder poisoned the lock: each write is whole. + fn probes(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.0.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Record a scrape's latency counters and return the latency series it publishes. + #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] + pub(crate) fn record_latency( + &self, + key: &str, + observations: &[Observation], + pool: Option<&str>, + now: Instant, + ) -> Vec { + // Built before locking, so the store is held only to append it. + let snapshot = crate::latency::Snapshot::of(observations, pool); + // The counters carry no pool label, so an EPP serving several pools proves nothing. + let attributable = !serves_several_pools(observations); + let mut probes = self.probes(); + let probe = probes.entry(key.to_owned()).or_default(); + if attributable + && probe + .latency + .latest() + .is_some_and(|earlier| snapshot.produced_since(earlier)) + { + probe.last_progress = Some(now); + } + probe.latency.record(snapshot, now) + } + + /// Record a successful scrape at `now`, with `missing` naming the series it lacked. + #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] + #[expect(clippy::too_many_arguments, reason = "one scrape's count, gap, signals, and time")] + pub(crate) fn record_success( + &self, + key: &str, + ready_endpoints: Option, + missing: Option, + observations: Vec, + now: Instant, + ) { + let mut probes = self.probes(); + let probe = probes.entry(key.to_owned()).or_default(); + probe.first_attempt.get_or_insert(now); + match ready_endpoints { + Some(count) if count < 1.0 => { + probe.zero_streak = probe.zero_streak.saturating_add(1); + probe.ready_streak = 0; + probe.busy = probe + .last_progress + .is_some_and(|at| now.saturating_duration_since(at) <= PROGRESS_WINDOW); + probe.no_endpoints |= probe.zero_streak >= STREAK && !probe.busy; + }, + Some(_) => { + probe.ready_streak = probe.ready_streak.saturating_add(1); + probe.zero_streak = 0; + probe.busy = false; + probe.no_endpoints &= probe.ready_streak < STREAK; + }, + // No count to read: reachable is all that is known. + None => { + probe.zero_streak = 0; + probe.ready_streak = 0; + probe.busy = false; + probe.no_endpoints = false; + }, + } + probe.last_good = Some(now); + probe.ready_endpoints = ready_endpoints; + probe.unpublished = Some(observations); + probe.failure = None; + probe.missing = ready_endpoints.is_none().then_some(missing).flatten(); + } + + /// Record a failed scrape at `now`, keeping what the last good one saw. + #[expect(clippy::significant_drop_tightening, reason = "the guard covers one whole update")] + pub(crate) fn record_failure(&self, key: &str, class: ScrapeClass, now: Instant) { + let mut probes = self.probes(); + let probe = probes.entry(key.to_owned()).or_default(); + probe.first_attempt.get_or_insert(now); + probe.failure = Some(class); + } + + /// The verdict for `key` at `now`. + /// + /// `AwaitingFirstScrape`, which is `Unknown` and publishes nothing, until the + /// provider has been attempted and through the first `stale_after` while no + /// scrape has succeeded yet, so a starting operator states that it is waiting. + #[expect( + clippy::significant_drop_tightening, + reason = "the guard covers one small evaluation" + )] + pub(crate) fn verdict(&self, key: &str, unavailable: bool, stale_after: Duration, now: Instant) -> Option { + let awaiting = Verdict { + reason: Reason::AwaitingFirstScrape, + message: "awaiting the first metrics scrape".to_owned(), + }; + let probes = self.probes(); + let Some(probe) = probes.get(key) else { + return (!unavailable).then_some(awaiting); + }; + let within = |at: Instant| now.saturating_duration_since(at) <= stale_after; + if probe.last_good.is_none() && !unavailable && probe.first_attempt.is_some_and(within) { + return Some(awaiting); + } + Some(evaluate( + probe, + probe.last_good.is_some_and(within), + unavailable, + stale_after, + )) + } + + /// The signals of the latest successful scrape, once: stale load is never republished. + pub(crate) fn take_fresh(&self, key: &str) -> Option> { + self.probes().get_mut(key)?.unpublished.take() + } + + /// Ready endpoints in the provider's last successful scrape, `None` when it exposed no count. + pub(crate) fn ready_endpoints(&self, key: &str) -> Option { + self.probes().get(key)?.ready_endpoints + } + + /// Forget providers not in `keep`. + pub(crate) fn retain(&self, keep: &std::collections::BTreeSet) { + self.probes().retain(|key, _| { + let kept = keep.contains(key); + if !kept && let Some((_, identity)) = key.split_once('/') { + crate::metrics::forget_provider_scrapes(identity); + } + kept + }); + } +} + +/// The verdict for one probe. `fresh` says whether its last success is within `stale_after`. +fn evaluate(probe: &Probe, fresh: bool, unavailable: bool, stale_after: Duration) -> Verdict { + if unavailable { + return Verdict { + reason: Reason::ProviderUnavailable, + message: "provider is Unavailable".to_owned(), + }; + } + if !fresh { + let window = stale_after.as_secs(); + return match probe.failure { + Some(class) => Verdict { + reason: class.reason(), + message: format!( + "no successful metrics scrape in the last {window}s; the latest failed: {}", + class.as_str() + ), + }, + None => Verdict { + reason: Reason::MetricsStale, + message: format!("no successful metrics scrape in the last {window}s"), + }, + }; + } + from_ready_endpoints(probe) +} + +/// The verdict for a freshly scraped provider, from its ready-endpoint count. +fn from_ready_endpoints(probe: &Probe) -> Verdict { + let count = probe.ready_endpoints; + if probe.no_endpoints { + let message = match count { + Some(count) if count >= 1.0 => format!("{count} ready endpoints, awaiting a second scrape"), + _ => format!("0 ready endpoints for {STREAK} or more scrapes"), + }; + return Verdict { + reason: Reason::NoEndpointsReady, + message, + }; + } + let Some(count) = count else { + return Verdict { + reason: Reason::NoLivenessCheck, + message: probe + .missing + .clone() + .unwrap_or_else(|| "metrics reachable, but no ready-endpoint series".to_owned()), + }; + }; + Verdict { + reason: Reason::Ready, + message: ready_message(count, probe.busy), + } +} + +/// The detail for a ready provider counting `count` endpoints. +fn ready_message(count: f64, busy: bool) -> String { + if count >= 1.0 { + format!("{count} ready endpoints") + } else if busy { + format!( + "0 ready endpoints, but the engine answered in the last {}s", + PROGRESS_WINDOW.as_secs() + ) + } else { + "0 ready endpoints in the latest scrape only".to_owned() + } +} + +/// The condition type this module owns. +pub const READY_CONDITION: &str = "Ready"; + +/// The `Ready` condition to write, or `None` when `current` already says the same. +/// +/// A changed message alone is not written, so a moving endpoint count does not +/// churn status. A new generation is. `lastTransitionTime` moves only when the status does. +pub(crate) fn ready_condition( + current: &[Condition], + verdict: &Verdict, + now_rfc3339: &str, + generation: Option, +) -> Option { + let held = current.iter().find(|c| c.type_ == READY_CONDITION); + let status = verdict.reason.status(); + if held.is_some_and(|c| { + c.status == status && c.reason == verdict.reason.as_str() && c.observed_generation == generation + }) { + return None; + } + let last_transition_time = held + .filter(|c| c.status == status) + .map_or_else(|| now_rfc3339.to_owned(), |c| c.last_transition_time.clone()); + Some(Condition { + type_: READY_CONDITION.to_owned(), + status: status.to_owned(), + reason: verdict.reason.as_str().to_owned(), + message: verdict.message.clone(), + last_transition_time, + observed_generation: generation, + }) +} + +/// Ready endpoints for `pool` in `observations`, from the first of `names` present. +/// +/// With a pool, only series whose `name` label is that pool count, so an EPP that +/// serves several pools is read for the configured one. +pub(crate) fn ready_endpoints(observations: &[Observation], names: &[&str], pool: Option<&str>) -> Option { + names.iter().find_map(|name| { + observations + .iter() + .filter(|o| o.metric == *name) + .filter(|o| pool.is_none_or(|pool| o.labels.get("name").is_some_and(|n| n == pool))) + .map(|o| o.value) + .reduce(f64::max) + }) +} + +/// Where a provider's in-flight count came from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum InFlightSource { + /// The EPP's inflight-load-producer, at or above the pool averages. + Epp, + /// The pool averages of running and queued times ready endpoints, above the EPP count or + /// in place of it. + EngineAverages, +} + +/// Requests the provider holds: the larger of the EPP's per-endpoint count summed and its +/// pool averages of running and queued times `ready`, plus requests flow control holds. +/// +/// The larger of the two, so an EPP restart that zeroes its count does not read as idle. The +/// per-endpoint count carries no pool label, so it is skipped when the EPP serves several +/// pools. `None` when neither source is present, or the site has no fresh endpoint behind +/// frozen averages. +pub(crate) fn in_flight( + observations: &[Observation], + ready: Option, + pool: Option<&str>, +) -> Option<(f64, InFlightSource)> { + let counted = (!serves_several_pools(observations)) + .then(|| epp_count(observations)) + .flatten(); + // No fresh endpoint means the averages are frozen at their last value: unknown, not idle. + let averaged = ready.filter(|ready| *ready > 0.0).and_then(|ready| { + EPP_AVERAGES + .iter() + .map(|names| ready_endpoints(observations, names, pool)) + .sum::>() + .map(|per_endpoint| per_endpoint * ready) + }); + let (held, source) = match (counted, averaged) { + (Some(counted), Some(averaged)) if averaged > counted => (averaged, InFlightSource::EngineAverages), + (Some(counted), _) => (counted, InFlightSource::Epp), + (None, Some(averaged)) => (averaged, InFlightSource::EngineAverages), + (None, None) => return None, + }; + Some((held + flow_control_queued(observations, pool), source)) +} + +/// The EPP's in-flight count summed over endpoints, each endpoint's largest across producer +/// instances so two producers do not double it. `None` when the EPP exports none. +fn epp_count(observations: &[Observation]) -> Option { + let mut producers: std::collections::BTreeMap<&str, std::collections::BTreeMap<(&str, &str), f64>> = + std::collections::BTreeMap::new(); + for observation in observations.iter().filter(|o| o.metric == EPP_IN_FLIGHT) { + let label = |name: &str| observation.labels.get(name).map_or("", String::as_str); + // Series per fairness and priority add up; producer instances repeat the same requests. + *producers + .entry(label("producer_name")) + .or_default() + .entry((label("namespace"), label("endpoint_name"))) + .or_insert(0.0) += observation.value; + } + let mut peak: std::collections::BTreeMap<(&str, &str), f64> = std::collections::BTreeMap::new(); + for counts in producers.values() { + for (endpoint, count) in counts { + let held = peak.entry(*endpoint).or_insert(0.0); + *held = held.max(*count); + } + } + (!peak.is_empty()).then(|| peak.values().sum()) +} + +/// Whether the EPP reports more than one pool or namespace, so its unlabeled per-endpoint +/// count cannot be attributed to one pool. +fn serves_several_pools(observations: &[Observation]) -> bool { + let distinct = |metrics: &[&str], label: &str| { + observations + .iter() + .filter(|o| metrics.contains(&o.metric.as_str())) + .filter_map(|o| o.labels.get(label)) + .collect::>() + .len() + }; + distinct(&DEFAULT_READY_ENDPOINTS, "name") > 1 || distinct(&[EPP_IN_FLIGHT], "namespace") > 1 +} + +/// Requests the EPP's flow control holds for `pool`, 0 when it exports none. +fn flow_control_queued(observations: &[Observation], pool: Option<&str>) -> f64 { + observations + .iter() + .filter(|o| o.metric == EPP_FLOW_CONTROL_QUEUE) + .filter(|o| pool.is_none_or(|pool| o.labels.get("inference_pool").is_some_and(|p| p == pool))) + .map(|o| o.value) + .sum() +} + +#[cfg(test)] +#[expect(clippy::allow_attributes, reason = "blanket test suppressions")] +#[allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::too_many_lines, + clippy::shadow_unrelated, + reason = "tests" +)] +mod tests { + use std::collections::BTreeMap; + + use super::*; + + const STALE: Duration = Duration::from_secs(15); + + fn sample(metric: &str, pool: &str, value: f64) -> Observation { + Observation { + metric: metric.to_owned(), + labels: BTreeMap::from([("name".to_owned(), pool.to_owned())]), + value, + timestamp_ms: None, + } + } + + #[test] + fn ready_endpoints_prefers_the_current_name_and_reads_the_configured_pool() { + let observations = [ + sample("inference_pool_ready_pods", "qwen3", 4.0), + sample("llm_d_epp_ready_endpoints", "other", 9.0), + sample("llm_d_epp_ready_endpoints", "qwen3", 2.0), + ]; + assert_eq!( + ready_endpoints(&observations, &DEFAULT_READY_ENDPOINTS, Some("qwen3")), + Some(2.0) + ); + assert_eq!( + ready_endpoints(&observations[..1], &DEFAULT_READY_ENDPOINTS, Some("qwen3")), + Some(4.0), + "falls back to the deprecated name" + ); + assert_eq!( + ready_endpoints(&observations, &DEFAULT_READY_ENDPOINTS, Some("absent")), + None + ); + } + + fn reason(store: &ReadinessStore, now: Instant) -> Option { + store.verdict("p", false, STALE, now).map(|verdict| verdict.reason) + } + + #[test] + fn verdict_follows_the_latest_scrape_and_staleness() { + let store = ReadinessStore::default(); + let start = Instant::now(); + assert_eq!( + reason(&store, start), + Some(Reason::AwaitingFirstScrape), + "never attempted" + ); + + store.record_success("p", Some(2.0), None, vec![sample("q", "p", 1.0)], start); + assert_eq!(reason(&store, start), Some(Reason::Ready)); + assert_eq!(store.take_fresh("p").map(|held| held.len()), Some(1)); + assert_eq!(store.take_fresh("p"), None, "fresh signals are published once"); + + let missing = "metrics reachable, but no llm_d_epp_ready_endpoints series for poolName qwen3"; + store.record_success("p", None, Some(missing.to_owned()), Vec::new(), start); + let verdict = store.verdict("p", false, STALE, start).expect("judged"); + assert_eq!( + (verdict.reason, verdict.message.as_str()), + (Reason::NoLivenessCheck, missing), + "a 200 without the pool's series is not ready, and says which series" + ); + assert!(verdict.reason.excludes()); + assert_eq!(store.take_fresh("p").map(|held| held.len()), Some(0)); + + store.record_failure("p", ScrapeClass::Connect, start + STALE); + assert_eq!( + reason(&store, start + STALE), + Some(Reason::NoLivenessCheck), + "a failure within the window keeps the last verdict" + ); + assert_eq!(store.take_fresh("p"), None, "a failure publishes no held load"); + let stale = store + .verdict("p", false, STALE, start + STALE + Duration::from_secs(1)) + .expect("judged"); + assert_eq!(stale.reason, Reason::ScrapeFailed); + assert!( + stale.message.ends_with("the latest failed: connect"), + "{}", + stale.message + ); + } + + #[test] + fn zero_endpoints_take_two_scrapes_to_exclude_and_two_to_readmit() { + let store = ReadinessStore::default(); + let now = Instant::now(); + let scrape = |count| { + store.record_success("p", Some(count), None, Vec::new(), now); + reason(&store, now) + }; + assert_eq!(scrape(0.0), Some(Reason::Ready), "one zero reading is not enough"); + assert_eq!(scrape(0.0), Some(Reason::NoEndpointsReady)); + assert_eq!( + scrape(1.0), + Some(Reason::NoEndpointsReady), + "one ready reading is not enough" + ); + assert_eq!( + scrape(0.0), + Some(Reason::NoEndpointsReady), + "a relapse restarts the count" + ); + assert_eq!(scrape(1.0), Some(Reason::NoEndpointsReady)); + assert_eq!(scrape(1.0), Some(Reason::Ready)); + } + + /// One EPP scrape for pool qwen3: its ready count, a running average frozen at 128, and + /// `answered` usage reports so far. + fn saturated(ready: f64, answered: f64) -> Vec { + let unlabeled = |metric: &str, value: f64| Observation { + metric: metric.to_owned(), + labels: BTreeMap::new(), + value, + timestamp_ms: None, + }; + vec![ + sample("llm_d_epp_ready_endpoints", "qwen3", ready), + sample("llm_d_epp_average_running_requests", "qwen3", 128.0), + unlabeled("llm_d_epp_request_input_tokens_count", answered), + unlabeled("llm_d_epp_request_input_tokens_sum", answered * 100.0), + ] + } + + /// Record `observations` as the signals loop does, and return the verdict. + fn scrape_at(store: &ReadinessStore, observations: &[Observation], now: Instant) -> Option { + let ready = ready_endpoints(observations, &DEFAULT_READY_ENDPOINTS, Some("qwen3")); + store.record_latency("p", observations, Some("qwen3"), now); + store.record_success("p", ready, None, observations.to_vec(), now); + reason(store, now) + } + + const STEP: Duration = Duration::from_secs(5); + + #[test] + fn a_saturated_pool_that_still_answers_stays_ready_on_zero_counts() { + let store = ReadinessStore::default(); + let start = Instant::now(); + assert_eq!(scrape_at(&store, &saturated(2.0, 100.0), start), Some(Reason::Ready)); + for (n, answered) in [(1_u32, 140.0), (2, 180.0), (3, 220.0)] { + assert_eq!( + scrape_at(&store, &saturated(0.0, answered), start + STEP * n), + Some(Reason::Ready), + "scrape {n}: running 128 and answering is busy, not down" + ); + } + let verdict = store.verdict("p", false, STALE, start + STEP * 3).unwrap(); + assert!(verdict.message.contains("answered"), "{}", verdict.message); + } + + #[test] + fn a_zero_count_with_no_answers_excludes_even_with_running_requests() { + let store = ReadinessStore::default(); + let start = Instant::now(); + assert_eq!(scrape_at(&store, &saturated(2.0, 100.0), start), Some(Reason::Ready)); + assert_eq!( + scrape_at(&store, &saturated(0.0, 100.0), start + STEP), + Some(Reason::Ready) + ); + assert_eq!( + scrape_at(&store, &saturated(0.0, 100.0), start + STEP * 2), + Some(Reason::NoEndpointsReady), + "the running average freezes at zero endpoints, so it is no evidence" + ); + } + + #[test] + fn a_pool_that_stops_answering_is_excluded_once_the_window_passes() { + let store = ReadinessStore::default(); + let start = Instant::now(); + scrape_at(&store, &saturated(2.0, 100.0), start); + let answered = start + STEP; + scrape_at(&store, &saturated(0.0, 140.0), answered); + // Six more scrapes reach 30s after the last answer: still inside the window. + for n in 1..=6_u32 { + assert_eq!( + scrape_at(&store, &saturated(0.0, 140.0), answered + STEP * n), + Some(Reason::Ready), + "{}s after the last answer", + (STEP * n).as_secs() + ); + } + assert_eq!( + scrape_at(&store, &saturated(0.0, 140.0), answered + STEP * 7), + Some(Reason::NoEndpointsReady) + ); + } + + #[test] + fn answers_from_an_epp_serving_several_pools_prove_nothing() { + let store = ReadinessStore::default(); + let start = Instant::now(); + let with_other_pool = |ready, answered| { + let mut observations = saturated(ready, answered); + observations.push(sample("llm_d_epp_ready_endpoints", "llama", 3.0)); + observations + }; + scrape_at(&store, &with_other_pool(2.0, 100.0), start); + scrape_at(&store, &with_other_pool(0.0, 140.0), start + STEP); + assert_eq!( + scrape_at(&store, &with_other_pool(0.0, 180.0), start + STEP * 2), + Some(Reason::NoEndpointsReady) + ); + } + + #[test] + fn a_first_scrape_that_fails_gets_the_staleness_window_before_a_verdict() { + let store = ReadinessStore::default(); + let start = Instant::now(); + store.record_failure("p", ScrapeClass::Timeout, start); + assert_eq!( + reason(&store, start + STALE), + Some(Reason::AwaitingFirstScrape), + "grace, not a guess" + ); + assert_eq!( + reason(&store, start + STALE + Duration::from_secs(1)), + Some(Reason::ScrapeTimedOut) + ); + } + + #[test] + fn a_failed_scrape_names_its_class() { + let cases = [ + (ScrapeClass::Timeout, Reason::ScrapeTimedOut), + (ScrapeClass::Unauthorized, Reason::ScrapeUnauthorized), + (ScrapeClass::Tls, Reason::TlsHandshakeFailed), + (ScrapeClass::Dns, Reason::ScrapeFailed), + (ScrapeClass::Connect, Reason::ScrapeFailed), + (ScrapeClass::Http, Reason::ScrapeFailed), + (ScrapeClass::BodyCap, Reason::ScrapeFailed), + (ScrapeClass::Parse, Reason::ScrapeFailed), + (ScrapeClass::Config, Reason::ScrapeFailed), + ]; + let start = Instant::now(); + for (class, expected) in cases { + let store = ReadinessStore::default(); + store.record_failure("p", class, start); + let verdict = store + .verdict("p", false, STALE, start + STALE + Duration::from_secs(1)) + .expect("judged"); + assert_eq!(verdict.reason, expected, "{class:?}"); + assert!( + verdict.message.ends_with(class.as_str()), + "{class:?}: {}", + verdict.message + ); + assert!(verdict.reason.excludes(), "{class:?}"); + } + assert_eq!( + Reason::TlsHandshakeFailed.as_str(), + "TLSHandshakeFailed", + "the contract's spelling" + ); + } + + #[test] + fn providers_are_keyed_per_network() { + let store = ReadinessStore::default(); + let now = Instant::now(); + for _ in 0..STREAK { + store.record_success(&key("east", "p"), Some(0.0), None, Vec::new(), now); + } + store.record_success(&key("west", "p"), Some(3.0), None, Vec::new(), now); + let reason_in = |network| { + store + .verdict(&key(network, "p"), false, STALE, now) + .map(|verdict| verdict.reason) + }; + assert_eq!(reason_in("east"), Some(Reason::NoEndpointsReady)); + assert_eq!(reason_in("west"), Some(Reason::Ready)); + } + + #[test] + fn an_unavailable_provider_is_not_ready_whatever_its_metrics_say() { + let store = ReadinessStore::default(); + let now = Instant::now(); + store.record_success("p", Some(3.0), None, Vec::new(), now); + let verdict = store.verdict("p", true, STALE, now).unwrap(); + assert_eq!(verdict.reason, Reason::ProviderUnavailable); + assert!(verdict.reason.excludes()); + } + + #[test] + fn the_status_column_reads_like_a_node() { + assert_eq!(Reason::Ready.display(), "Ready"); + assert_eq!(Reason::MetricsNotConfigured.display(), "Unknown"); + assert_eq!(Reason::AwaitingFirstScrape.display(), "Unknown"); + for reason in [ + Reason::NoEndpointsReady, + Reason::NoLivenessCheck, + Reason::MetricsStale, + Reason::ScrapeFailed, + Reason::ProviderUnavailable, + ] { + assert_eq!(reason.display(), "NotReady", "{reason:?}"); + } + } + + #[test] + fn the_condition_is_written_on_a_status_or_reason_change_only() { + let ready = Verdict { + reason: Reason::Ready, + message: "2 ready endpoints".to_owned(), + }; + let first = ready_condition(&[], &ready, "t0", Some(3)).expect("absent is written"); + assert_eq!( + (first.status.as_str(), first.last_transition_time.as_str()), + ("True", "t0") + ); + + let recount = Verdict { + message: "3 ready endpoints".to_owned(), + ..ready + }; + assert!( + ready_condition(std::slice::from_ref(&first), &recount, "t1", Some(3)).is_none(), + "a new count alone is not news" + ); + let regenerated = ready_condition(std::slice::from_ref(&first), &recount, "t1", Some(4)) + .expect("a new generation is written"); + assert_eq!( + regenerated.last_transition_time, "t0", + "the same status keeps its transition time" + ); + + let down = Verdict { + reason: Reason::NoEndpointsReady, + message: "0 ready endpoints".to_owned(), + }; + let second = ready_condition(std::slice::from_ref(&first), &down, "t2", Some(3)).unwrap(); + assert_eq!( + (second.status.as_str(), second.last_transition_time.as_str()), + ("False", "t2") + ); + + let unreachable = Verdict { + reason: Reason::ScrapeFailed, + message: "no scrape".to_owned(), + }; + let third = ready_condition(std::slice::from_ref(&second), &unreachable, "t3", Some(3)).unwrap(); + assert_eq!( + third.last_transition_time, "t2", + "a new reason under the same status keeps the transition time" + ); + } + + #[test] + fn statuses_match_the_condition_contract() { + assert_eq!(Reason::Ready.status(), "True"); + assert_eq!(Reason::MetricsNotConfigured.status(), "Unknown"); + assert!(!Reason::MetricsNotConfigured.excludes(), "unknown is not down"); + assert_eq!(Reason::AwaitingFirstScrape.status(), "Unknown"); + assert!(!Reason::AwaitingFirstScrape.excludes(), "waiting is not down"); + for reason in [ + Reason::NoEndpointsReady, + Reason::MetricsStale, + Reason::NoLivenessCheck, + Reason::ScrapeTimedOut, + Reason::ScrapeUnauthorized, + Reason::TlsHandshakeFailed, + Reason::ScrapeFailed, + Reason::ProviderUnavailable, + ] { + assert_eq!(reason.status(), "False"); + assert!(reason.excludes()); + } + } + + fn counted(endpoint: &str, namespace: &str, producer: &str, value: f64) -> Observation { + Observation { + metric: "llm_d_epp_inflight_requests".to_owned(), + labels: BTreeMap::from([ + ("endpoint_name".to_owned(), endpoint.to_owned()), + ("namespace".to_owned(), namespace.to_owned()), + ("producer_name".to_owned(), producer.to_owned()), + ]), + value, + timestamp_ms: None, + } + } + + fn queued(pool: &str, value: f64) -> Observation { + Observation { + metric: "llm_d_epp_flow_control_queue_size".to_owned(), + labels: BTreeMap::from([("inference_pool".to_owned(), pool.to_owned())]), + value, + timestamp_ms: None, + } + } + + #[test] + fn in_flight_reads_the_larger_of_the_epp_count_and_the_pool_averages() { + let averages = [ + sample("llm_d_epp_average_running_requests", "qwen3", 6.0), + sample("llm_d_epp_average_queue_size", "qwen3", 1.5), + ]; + let mut observations = vec![counted("a", "ns", "p", 7.0), counted("b", "ns", "p", 5.0)]; + assert_eq!( + in_flight(&observations, Some(2.0), None), + Some((12.0, InFlightSource::Epp)), + "the EPP's count alone" + ); + observations.extend(averages.clone()); + assert_eq!( + in_flight(&observations, Some(1.0), Some("qwen3")), + Some((12.0, InFlightSource::Epp)), + "12 counted is above (6 + 1.5) x 1" + ); + let restarted = [vec![counted("a", "ns", "p", 0.0)], averages.to_vec()].concat(); + assert_eq!( + in_flight(&restarted, Some(2.0), Some("qwen3")), + Some((15.0, InFlightSource::EngineAverages)), + "an EPP restart reading 0 is floored by (6 + 1.5) x 2" + ); + assert_eq!( + in_flight(&averages, Some(2.0), Some("qwen3")), + Some((15.0, InFlightSource::EngineAverages)), + "the averages alone" + ); + } + + #[test] + fn in_flight_counts_each_endpoint_once_across_producers() { + let observations = [ + counted("a", "ns", "first", 4.0), + counted("a", "ns", "second", 3.0), + counted("b", "ns", "first", 2.0), + ]; + assert_eq!( + in_flight(&observations, None, None), + Some((6.0, InFlightSource::Epp)), + "4 + 2, not 9" + ); + } + + #[test] + fn in_flight_adds_requests_flow_control_holds_for_the_pool() { + let observations = [counted("a", "ns", "p", 3.0), queued("qwen3", 4.0), queued("other", 9.0)]; + assert_eq!( + in_flight(&observations, None, Some("qwen3")), + Some((7.0, InFlightSource::Epp)), + "3 dispatched plus 4 held for qwen3, not other's 9" + ); + } + + #[test] + fn in_flight_uses_pool_averages_when_the_epp_serves_several_pools() { + let observations = [ + counted("a", "ns", "p", 50.0), + sample("llm_d_epp_ready_endpoints", "qwen3", 2.0), + sample("llm_d_epp_ready_endpoints", "llama", 1.0), + sample("llm_d_epp_average_running_requests", "qwen3", 3.0), + sample("llm_d_epp_average_queue_size", "qwen3", 1.0), + ]; + assert_eq!( + in_flight(&observations, Some(2.0), Some("qwen3")), + Some((8.0, InFlightSource::EngineAverages)), + "the unlabeled count covers both pools, so (3 + 1) x 2 for qwen3" + ); + } + + #[test] + fn in_flight_is_unknown_rather_than_idle() { + let averages = [ + sample("llm_d_epp_average_running_requests", "qwen3", 6.0), + sample("llm_d_epp_average_queue_size", "qwen3", 1.5), + ]; + assert_eq!(in_flight(&averages, None, None), None, "no endpoint count, no estimate"); + assert_eq!( + in_flight(&averages, Some(0.0), Some("qwen3")), + None, + "no fresh endpoint behind frozen averages" + ); + assert_eq!( + in_flight(&averages[..1], Some(2.0), None), + None, + "a missing average is not zero" + ); + assert_eq!( + in_flight(&[queued("qwen3", 4.0)], None, Some("qwen3")), + None, + "a queue alone is no count" + ); + } +} diff --git a/operator/src/resources/consumer_config.rs b/operator/src/resources/consumer_config.rs index c4012f463..dcf24c1fa 100644 --- a/operator/src/resources/consumer_config.rs +++ b/operator/src/resources/consumer_config.rs @@ -603,6 +603,7 @@ mod tests { network: "test-net".to_owned(), local_site: "site-a".to_owned(), candidates, + excluded: Vec::new(), selection_policy: None, generated_at: None, } @@ -1008,6 +1009,7 @@ mod tests { network: "n".to_owned(), local_site: String::new(), candidates: vec![], + excluded: Vec::new(), selection_policy: None, generated_at: None, }; diff --git a/operator/src/resources/overlay_bridge.rs b/operator/src/resources/overlay_bridge.rs index c79849c22..d1c1ff1d0 100644 --- a/operator/src/resources/overlay_bridge.rs +++ b/operator/src/resources/overlay_bridge.rs @@ -118,6 +118,7 @@ mod tests { capacity_weight: 1, }) .collect(), + excluded: Vec::new(), selection_policy: None, generated_at: None, } diff --git a/operator/src/resources/overlay_envelope.rs b/operator/src/resources/overlay_envelope.rs index eb4e9acaf..85b78e764 100644 --- a/operator/src/resources/overlay_envelope.rs +++ b/operator/src/resources/overlay_envelope.rs @@ -300,6 +300,7 @@ mod tests { RoutingOverlay { network: "test-net".to_owned(), local_site: "site-a".to_owned(), + excluded: Vec::new(), candidates: vec![RoutingCandidate { kind: "inference_model".to_owned(), name: "model-a".to_owned(), @@ -330,6 +331,7 @@ mod tests { RoutingOverlay { network: "glb-demo".to_owned(), local_site: "east-edge".to_owned(), + excluded: Vec::new(), candidates: vec![ RoutingCandidate { kind: "inference_model".to_owned(), diff --git a/operator/src/resources/provider_metrics.rs b/operator/src/resources/provider_metrics.rs index 705792a65..9a064149e 100644 --- a/operator/src/resources/provider_metrics.rs +++ b/operator/src/resources/provider_metrics.rs @@ -107,88 +107,182 @@ pub(crate) struct CollectedMetrics { // Signals collection (poll mode) // --------------------------------------------------------------------------- -/// Scrape each provider's endpoint and keep only its configured coarse signals. +/// Scrape each provider's endpoint, recording its coarse signals and readiness. /// /// Sibling of [`collect_provider_metrics_with_refresh_interval`], which parses /// the same text into [`scoring::BackendMetrics`] for local scoring. This keeps /// the provider's own exposition narrowed to its declared `signalNames`, so the /// wire carries a coarse rollup rather than the full `/metrics` firehose. Fails /// closed on TLS: a provider whose TLS will not resolve is skipped, never -/// scraped in plaintext. A failed scrape leaves the last value to expire. +/// scraped in plaintext, and counts as a failed scrape. pub(crate) async fn collect_provider_signals( network_name: &str, providers: &[InferenceProvider], client: Option<&kube::Client>, -) -> HashMap> { - let mut out = HashMap::new(); + readiness: &crate::readiness::ReadinessStore, +) { for provider in providers { if provider.spec.grid_network_ref != network_name { continue; } - if let Some((identity, observations)) = scrape_provider_signals(provider, client).await { - out.insert(identity, observations); + let Some(plan) = signal_scrape_plan(provider) else { + continue; + }; + let key = crate::readiness::key(network_name, plan.identity); + match scrape_provider_signals(provider, &plan, client).await { + Ok(text) => record_scrape(readiness, &key, provider, &plan, &text), + Err(class) => { + crate::metrics::record_provider_scrape(plan.identity, class.as_str()); + readiness.record_failure(&key, class, Instant::now()); + }, } } - out } -/// Scrape one provider's coarse signals, or `None` to leave its last value be. -/// -/// Every skip and failure returns `None`: a provider absent from the collection -/// is left alone rather than erased, so a missed scrape expires on its own. +/// Record one successful scrape: its ready-endpoint count and its declared signals. +fn record_scrape( + readiness: &crate::readiness::ReadinessStore, + key: &str, + provider: &InferenceProvider, + plan: &SignalScrapePlan<'_>, + text: &str, +) { + // Every sample feeds this site's own windows; only gauges are republished. + let (parsed, republishable): (Vec<_>, Vec<_>) = crate::signals::parse_scrape(text).into_iter().unzip(); + let pool = provider + .spec + .metrics_config + .as_ref() + .and_then(|mc| mc.pool_name.as_deref()); + let ready = crate::readiness::ready_endpoints(&parsed, plan.ready_names(), pool); + let missing = count_scrape(plan, pool, ready); + let in_flight = in_flight_observation(&parsed, ready, pool, plan.identity); + let latency = readiness.record_latency(key, &parsed, pool, Instant::now()); + let observations = parsed + .into_iter() + .zip(republishable) + .filter_map(|(o, republishable)| republishable.then_some(o)) + .filter(|o| plan.wanted.contains(o.metric.as_str())) + // A local sample's freshness is its collection time, so drop any trailing + // timestamp. Only relayed peer samples carry a per-sample stamp. + .map(|mut o| { + o.timestamp_ms = None; + o + }) + .chain(in_flight) + .chain(latency) + .collect(); + readiness.record_success(key, ready, missing, observations, Instant::now()); +} + +/// Count a scrape that answered: `success` with the pool's ready-endpoint series, else +/// `no_series`, returned as the message naming what was missing. +fn count_scrape(plan: &SignalScrapePlan<'_>, pool: Option<&str>, ready: Option) -> Option { + if ready.is_some() { + crate::metrics::record_provider_scrape(plan.identity, "success"); + crate::metrics::set_provider_last_scrape_success(plan.identity, std::time::SystemTime::now()); + return None; + } + crate::metrics::record_provider_scrape(plan.identity, "no_series"); + let series = plan.ready_names().join(" or "); + Some(match pool { + Some(pool) => format!("metrics reachable, but no {series} series for poolName {pool}"), + None => format!("metrics reachable, but no {series} series"), + }) +} + +/// The resolved `grid_provider_in_flight_requests` sample, logging which source it came from. +fn in_flight_observation( + parsed: &[crate::signals::Observation], + ready: Option, + pool: Option<&str>, + identity: &str, +) -> Option { + let (value, source) = crate::readiness::in_flight(parsed, ready, pool)?; + tracing::debug!( + provider = identity, + in_flight = value, + ?source, + "signals: provider in-flight resolved" + ); + Some(crate::signals::Observation { + metric: crate::readiness::IN_FLIGHT_SIGNAL.to_owned(), + labels: std::collections::BTreeMap::new(), + value, + timestamp_ms: None, + }) +} + +/// Scrape one provider's exposition, or `None` when TLS or the scrape fails. async fn scrape_provider_signals( provider: &InferenceProvider, + plan: &SignalScrapePlan<'_>, client: Option<&kube::Client>, -) -> Option<(String, Vec)> { - let mc = provider.spec.metrics_config.as_ref()?; - let (identity, url, wanted) = signal_scrape_plan(provider)?; +) -> Result { + let mc = provider + .spec + .metrics_config + .as_ref() + .ok_or(crate::readiness::ScrapeClass::Config)?; + let identity = plan.identity; let tls_config = match resolve_tls_config(mc.tls.as_ref(), client, identity).await { Ok(cfg) => cfg, Err((_reason, e)) => { if mc.tls.is_some() { - tracing::warn!(provider = identity, error = %e, "signals: provider metrics TLS unavailable; not scraping in plaintext"); + tracing::debug!(provider = identity, error = %e, "signals: provider metrics TLS unavailable; not scraping in plaintext"); } - return None; + return Err(crate::readiness::ScrapeClass::Tls); }, }; let timeout = parse_metrics_timeout(&mc.timeout); - let text = scrape_metrics(&url, timeout, tls_config, mc.auth.as_ref().zip(client)) + scrape_metrics(&plan.url, timeout, tls_config, mc.auth.as_ref().zip(client)) .await - .inspect_err(|e| { - tracing::debug!(provider = identity, error = %e, "signals: provider scrape failed; last value left to expire"); - }) - .ok()?; - let observations = crate::signals::parse(&text) - .into_iter() - .filter(|o| wanted.contains(o.metric.as_str())) - // A local sample's freshness is its collection time, so drop any trailing - // timestamp. Only relayed peer samples carry a per-sample stamp. - .map(|mut o| { - o.timestamp_ms = None; - o + .map_err(|e| { + tracing::debug!(provider = identity, error = %e, "signals: provider scrape failed"); + e.class() }) - .collect(); - Some((identity.to_owned(), observations)) } -/// The scrape target for a provider's coarse signals, if it is eligible. +/// What to scrape for one provider and which of its series to republish. +pub(crate) struct SignalScrapePlan<'provider> { + /// The provider's routing identity. + pub(crate) identity: &'provider str, + /// The metrics URL. + pub(crate) url: String, + /// Declared signal names republished as coarse signals; may be empty. + pub(crate) wanted: std::collections::BTreeSet, + /// A declared ready-endpoints metric, replacing the defaults. + ready_override: Option<&'provider str>, +} + +impl SignalScrapePlan<'_> { + /// The ready-endpoints names to read, preferred first. + fn ready_names(&self) -> &[&str] { + self.ready_override + .as_ref() + .map_or(&crate::readiness::DEFAULT_READY_ENDPOINTS[..], std::slice::from_ref) + } +} + +/// The scrape plan for a provider, if it is eligible. /// /// Pure and synchronous: eligibility is decided here so the scrape path stays /// the I/O alone. `None` for a provider with no metrics config, no routing -/// identity, a blank endpoint, or no declared signal names. -fn signal_scrape_plan(provider: &InferenceProvider) -> Option<(&str, String, std::collections::BTreeSet)> { +/// identity, or a blank endpoint. A provider with no signal names is still +/// scraped, for readiness. +pub(crate) fn signal_scrape_plan(provider: &InferenceProvider) -> Option> { let mc = provider.spec.metrics_config.as_ref()?; let identity = routing_identity(provider)?; let endpoint = provider.spec.endpoint.trim(); if endpoint.is_empty() || mc.metrics_endpoint.as_deref().is_some_and(|ep| ep.trim().is_empty()) { return None; } - let wanted = signal_metric_names(&mc.signal_names); - if wanted.is_empty() { - return None; - } - let url = metrics_url(mc.metrics_endpoint.as_deref().unwrap_or(endpoint), &mc.path); - Some((identity, url, wanted)) + Some(SignalScrapePlan { + identity, + url: metrics_url(mc.metrics_endpoint.as_deref().unwrap_or(endpoint), &mc.path), + wanted: signal_metric_names(&mc.signal_names), + ready_override: mc.signal_names.ready_endpoints.as_deref(), + }) } /// The source metric names a provider declares for its coarse signals. @@ -654,6 +748,7 @@ pub(crate) fn classify_scrape_error(err: &metrics_scraper::MetricsScrapeError) - }, metrics_scraper::MetricsScrapeError::Credential(_) | metrics_scraper::MetricsScrapeError::PlaintextCredential(_) => "MetricsCredentialUnavailable", + metrics_scraper::MetricsScrapeError::BodyTooLarge(_) => "MetricsBodyTooLarge", metrics_scraper::MetricsScrapeError::InvalidUrl(_) | metrics_scraper::MetricsScrapeError::NonOkStatus { .. } | metrics_scraper::MetricsScrapeError::Encoding(_) => "MetricsScrapeError", @@ -753,6 +848,107 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + /// A `KServe` EPP exposition for pool `qwen3-kserve`, in the label sets the lab EPP emits: + /// `served` streaming requests answered, about 104 usage reports each. + #[expect(clippy::too_many_lines, reason = "one exposition, line for line")] + fn kserve_epp_exposition(served: f64, ready: f64) -> String { + let l = r#"fairness_id="default-flow",model_name="qwen3",priority="0",target_model_name="qwen3""#; + let (b01, b05, b1) = (served * 0.2, served * 0.6, served * 0.9); + let (ttft_sum, tpot_sum) = (served * 0.4, served * 0.02); + let (reports, tokens) = (served * 104.0, served * 104.0 * 300.0); + let (requests, errors) = (served + 14.0, served / 100.0); + format!( + r#"# HELP llm_d_epp_ready_endpoints Ready endpoints. +# TYPE llm_d_epp_ready_endpoints gauge +llm_d_epp_ready_endpoints{{name="qwen3-kserve"}} {ready} +# TYPE llm_d_epp_average_running_requests gauge +llm_d_epp_average_running_requests{{name="qwen3-kserve"}} 128 +# TYPE llm_d_epp_request_ttft_seconds histogram +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="false",le="0.1"}} 14 +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="false",le="+Inf"}} 14 +llm_d_epp_request_ttft_seconds_sum{{{l},streaming="false"}} 1.9 +llm_d_epp_request_ttft_seconds_count{{{l},streaming="false"}} 14 +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="0.1"}} {b01} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="0.5"}} {b05} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="1"}} {b1} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="2.5"}} {served} +llm_d_epp_request_ttft_seconds_bucket{{{l},streaming="true",le="+Inf"}} {served} +llm_d_epp_request_ttft_seconds_sum{{{l},streaming="true"}} {ttft_sum} +llm_d_epp_request_ttft_seconds_count{{{l},streaming="true"}} {served} +# TYPE llm_d_epp_request_streaming_tpot_seconds histogram +llm_d_epp_request_streaming_tpot_seconds_sum{{{l}}} {tpot_sum} +llm_d_epp_request_streaming_tpot_seconds_count{{{l}}} {served} +# TYPE llm_d_epp_request_input_tokens histogram +llm_d_epp_request_input_tokens_sum{{{l}}} {tokens} +llm_d_epp_request_input_tokens_count{{{l}}} {reports} +# TYPE llm_d_epp_request_total counter +llm_d_epp_request_total{{{l}}} {requests} +# TYPE llm_d_epp_request_error_total counter +llm_d_epp_request_error_total{{{l},error_code="503"}} {errors} +"# + ) + } + + #[test] + #[expect(clippy::too_many_lines, reason = "two scrapes and every published series")] + fn a_kserve_epp_scrape_publishes_latency_and_republishes_gauges_only() { + let readiness = crate::readiness::ReadinessStore::default(); + let mut mc = mc_with_queue("llm_d_epp_average_running_requests"); + mc.pool_name = Some("qwen3-kserve".to_owned()); + // A counter declared as a signal is still never republished. + mc.signal_names.error_rate = Some("llm_d_epp_request_total".to_owned()); + let provider = provider_fixture("qwen3-hq-east", "http://epp:9090", Some(mc)); + let plan = signal_scrape_plan(&provider).unwrap_or_else(|| std::process::abort()); + record_scrape( + &readiness, + "net/p", + &provider, + &plan, + &kserve_epp_exposition(11_000.0, 2.0), + ); + assert!(readiness.take_fresh("net/p").is_some(), "first scrape recorded"); + record_scrape( + &readiness, + "net/p", + &provider, + &plan, + &kserve_epp_exposition(11_581.0, 0.0), + ); + let published = readiness.take_fresh("net/p").unwrap_or_default(); + let value = |name: &str| published.iter().find(|o| o.metric == name).map(|o| o.value); + for name in [ + crate::latency::TTFT_P50_SIGNAL, + crate::latency::TTFT_P90_SIGNAL, + crate::latency::TPOT_SIGNAL, + crate::latency::PREFILL_SIGNAL, + crate::latency::ERROR_RATIO_SIGNAL, + ] { + assert!(value(name).is_some(), "{name} missing from {published:?}"); + } + assert!((value(crate::latency::TPOT_SIGNAL).unwrap_or_default() - 0.02).abs() < 1e-9); + // 581 answers at 0.4s mean TTFT over 300 prompt tokens per usage report. + let prefill = value(crate::latency::PREFILL_SIGNAL).unwrap_or_default(); + assert!((prefill - 0.4 / 300.0).abs() < 1e-9, "prefill {prefill}"); + assert!( + published.iter().all(|o| !o.metric.ends_with("_total") + && !o.metric.ends_with("_count") + && !o.metric.ends_with("_sum") + && !o.metric.ends_with("_bucket")), + "counters and histogram parts stay local: {published:?}" + ); + assert!( + value("llm_d_epp_average_running_requests").is_some(), + "declared gauge republished" + ); + let verdict = readiness + .verdict("net/p", false, Duration::from_secs(15), Instant::now()) + .unwrap_or_else(|| std::process::abort()); + assert!( + verdict.message.contains("answered"), + "progress seen through the parser: {verdict:?}" + ); + } + fn mc_with_queue(metric_name: &str) -> MetricsConfig { MetricsConfig { path: "/metrics".to_owned(), @@ -874,6 +1070,7 @@ mod tests { prefix_cache_hit_ratio: Some("my_prefix".to_owned()), error_rate: Some("my_errors".to_owned()), healthy: Some("my_health".to_owned()), + ready_endpoints: None, }; let names = metric_names_from_config(&cfg, None, None); assert_eq!(names.queue_depth.as_deref(), Some("my_queue")); diff --git a/operator/src/resources/routing_overlay.rs b/operator/src/resources/routing_overlay.rs index e1468dcfc..43fc47a60 100644 --- a/operator/src/resources/routing_overlay.rs +++ b/operator/src/resources/routing_overlay.rs @@ -12,9 +12,9 @@ //! # Phase 1 / OP-01 semantics //! //! - [`GridSite`]s are used to resolve per-provider site membership via `spec.siteSelector.matchLabels`. An empty -//! selector matches all sites in the same [`GridNetwork`]. +//! selector means this site alone, when the network lists it. //! - Each `(model, site)` pair becomes one `RoutingCandidate`. -//! - `candidate.site` = the [`GridSite`] name (resolved via selector). +//! - `candidate.site` = the [`GridSite`]'s site id (resolved via selector), as peers are keyed. //! - `candidate.cluster` = `spec.routingClusterRef` when set, otherwise the [`InferenceProvider`] metadata name. The //! gateway uses this as the upstream cluster reference in its local routing configuration. //! - When no [`GridSite`]s are provided, the routing identity (`spec.routingClusterRef` or provider name) is used as @@ -48,6 +48,7 @@ use k8s_openapi::api::core::v1::ConfigMap; use serde::{Deserialize, Serialize}; use crate::{ + controller::grid_network::peer_site_key, crd::{ auth::{AccessPolicy, AuthStrategy}, grid_network::GridNetwork, @@ -931,6 +932,11 @@ pub struct RoutingOverlay { /// freshness, score, then alphabetical tiebreak. pub candidates: Vec, + /// Candidates excluded from routing, kept for the serving config so the gateway + /// can answer 503 for a known model. Never on the overlay's own wire. + #[serde(skip)] + pub excluded: Vec, + /// Optional explicit local selection policy for Praxis. An absent field is /// intentionally backward-compatible and means deterministic selection. #[serde(skip_serializing_if = "Option::is_none")] @@ -1118,7 +1124,7 @@ fn assign_selection_groups(candidates: &mut [RoutingCandidate], policy: crate::c /// Only [`InferenceProvider`]s whose `spec.gridNetworkRef` matches /// `network.metadata.name` are included. Each provider's /// `spec.siteSelector.matchLabels` is matched against the supplied -/// `sites`; an empty selector matches all sites in the network. +/// `sites`; an empty selector means `local_site` alone. /// /// The `local_site` parameter identifies this gateway's own site. /// Praxis uses it to score candidates running on the local site higher @@ -1272,7 +1278,7 @@ pub fn render_routing_overlay_with_admission( .find(|site| site.metadata.name.as_deref() == Some(local_site) && site.spec.grid_network_ref == network_name) .and_then(|site| site.metadata.labels.as_ref()); - let mut candidates = collect_candidates(network_name, sites, providers, consumer_site_labels)?; + let mut candidates = collect_candidates(network_name, sites, providers, consumer_site_labels, local_site)?; for provider in remote_crdt_providers { let access_policy = crdt_access_policy_to_operator(&provider.access_policy); let access_result = evaluate_access_policy(&access_policy, consumer_site_labels); @@ -1304,7 +1310,9 @@ pub fn render_routing_overlay_with_admission( } enrich_candidates(&mut candidates, local_site, sites, network_name, &admission_map); - candidates.retain(|c| c.admission_state != Some(AdmissionState::Excluded)); + let (excluded, mut candidates): (Vec<_>, Vec<_>) = candidates + .into_iter() + .partition(|c| c.admission_state == Some(AdmissionState::Excluded)); let policy = network .spec @@ -1381,6 +1389,7 @@ pub fn render_routing_overlay_with_admission( network: network_name.to_owned(), local_site: local_site.to_owned(), candidates, + excluded, selection_policy, generated_at: generated_at.map(str::to_owned), }) @@ -1454,6 +1463,7 @@ fn collect_candidates( sites: &[GridSite], providers: &[InferenceProvider], consumer_site_labels: Option<&BTreeMap>, + local_site: &str, ) -> Result, String> { // Pre-filter sites to those in this network. let network_sites: Vec<&GridSite> = sites @@ -1475,7 +1485,7 @@ fn collect_candidates( continue; } - let resolution = resolve_sites(provider, &network_sites); + let resolution = resolve_sites(provider, &network_sites, local_site); all.extend(candidates_from_provider(provider, &resolution)?); } Ok(all) @@ -1487,12 +1497,25 @@ fn collect_candidates( /// which enables the Phase 1 provider-name fallback. Returns /// [`SiteResolution::Known`] otherwise — with an empty `Vec` if the /// selector matched nothing, which suppresses candidate generation. -fn resolve_sites(provider: &InferenceProvider, network_sites: &[&GridSite]) -> SiteResolution { +/// +/// An empty selector means this site alone, when the inventory lists it, as the +/// provider controller attributes it. Sites are keyed as peers are, so a +/// discovered peer reads as its site id, not its object name. +fn resolve_sites(provider: &InferenceProvider, network_sites: &[&GridSite], local_site: &str) -> SiteResolution { if network_sites.is_empty() { return SiteResolution::Unavailable; } let selector = &provider.spec.site_selector.match_labels; + let key = |site: &&GridSite| peer_site_key(site).map(|(key, _)| key); + if selector.is_empty() { + let listed = network_sites.iter().filter_map(key).any(|site| site == local_site); + return SiteResolution::Known(if listed { + vec![local_site.to_owned()] + } else { + Vec::new() + }); + } let names: Vec = network_sites .iter() @@ -1502,7 +1525,7 @@ fn resolve_sites(provider: &InferenceProvider, network_sites: &[&GridSite]) -> S .iter() .all(|(k, v)| site_labels.is_some_and(|labels| labels.get(k).is_some_and(|sv| sv == v))) }) - .map(|site| site.metadata.name.clone().unwrap_or_else(|| "unknown-site".to_owned())) + .filter_map(key) .collect::>() .into_iter() .collect(); @@ -1982,9 +2005,19 @@ mod tests { .unwrap_or_else(|_| std::process::abort()) } + /// A label every labeled test site carries, so a provider can be placed at all of them. + const PLACEMENT: (&str, &str) = ("test.grid/placement", "all"); + + /// `provider` placed at every labeled test site, as an explicit selector does. + fn placed_everywhere(mut provider: InferenceProvider) -> InferenceProvider { + provider.spec.site_selector.match_labels = BTreeMap::from([(PLACEMENT.0.to_owned(), PLACEMENT.1.to_owned())]); + provider + } + fn test_site_with_labels(name: &str, network: &str, labels: &[(&str, &str)]) -> GridSite { let labels_map: serde_json::Map = labels .iter() + .chain(std::iter::once(&PLACEMENT)) .map(|(k, v)| (k.to_string(), serde_json::Value::String(v.to_string()))) .collect(); serde_json::from_value(serde_json::json!({ @@ -3284,30 +3317,55 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn empty_selector_matches_all_sites_in_network() { - let network = test_network("net"); - let site_a = test_site("site-a", "net"); - let site_b = test_site("site-b", "net"); - let provider = test_provider("prov", "net", &["model"]); + fn an_empty_selector_places_a_provider_at_the_local_site_only() { + let render = |local: &str| { + render_routing_overlay( + &test_network("net"), + &[test_site("site-a", "net"), test_site("site-b", "net")], + &[test_provider("prov", "net", &["model"])], + &[], + local, + None, + None, + &scoring::ScoringWeights::default(), + ) + .map(|overlay| overlay.candidates.into_iter().map(|c| c.site).collect::>()) + }; + assert!( + matches!(render("site-a"), Ok(sites) if sites == ["site-a"]), + "not every site in the network" + ); + assert!( + matches!(render("absent"), Ok(sites) if sites.is_empty()), + "a local site the inventory does not list hosts nothing" + ); + } + + #[test] + fn a_selector_names_a_discovered_peer_by_its_site_id() { + let mut peer = test_site_with_labels("grid-retail", "net", &[("hw", "gpu")]); + peer.metadata.labels.get_or_insert_with(BTreeMap::new).insert( + crate::controller::grid_network::LABEL_AUTO_DISCOVERED.to_owned(), + "true".to_owned(), + ); + peer.metadata.annotations = Some(BTreeMap::from([( + crate::controller::grid_network::ANNOTATION_SITE_ID.to_owned(), + "retail".to_owned(), + )])); + let provider = test_provider_with_selector("prov", "net", &["model"], &[("hw", "gpu")]); let overlay = render_routing_overlay( - &network, - &[site_a, site_b], + &test_network("net"), + &[peer], &[provider], &[], - "test-site", + "hq", None, None, &scoring::ScoringWeights::default(), ) .unwrap_or_else(|_| std::process::abort()); - assert_eq!( - overlay.candidates.len(), - 2, - "empty selector should produce one candidate per site" - ); let sites: Vec<&str> = overlay.candidates.iter().map(|c| c.site.as_str()).collect(); - assert!(sites.contains(&"site-a"), "site-a must be in candidates"); - assert!(sites.contains(&"site-b"), "site-b must be in candidates"); + assert_eq!(sites, ["retail"], "keyed as peers are, not by object name"); } #[test] @@ -3404,7 +3462,7 @@ mod tests { &[site], &[p1, p2], &[], - "test-site", + "site-a", None, None, &scoring::ScoringWeights::default(), @@ -3430,7 +3488,7 @@ mod tests { &[site], &[provider], &[], - "test-site", + "site-a", None, None, &scoring::ScoringWeights::default(), @@ -3700,13 +3758,17 @@ mod tests { &[site_a, site_b], &[provider], &[], - "test-site", + "site-a", None, None, &scoring::ScoringWeights::default(), ) .unwrap_or_else(|_| std::process::abort()); - assert_eq!(overlay.candidates.len(), 2, "one candidate per matched site"); + assert_eq!( + overlay.candidates.len(), + 1, + "an empty selector places it at the local site" + ); assert!( overlay.candidates.iter().all(|c| !c.fresh), "every (model, site) candidate from a Degraded provider must have fresh=false" @@ -6308,7 +6370,7 @@ mod tests { let network = test_network("net"); let site_prod = test_site_with_labels("site-prod", "net", &[("env", "prod")]); let site_staging = test_site_with_labels("site-staging", "net", &[("env", "staging")]); - let provider = test_provider("unrestricted-prov", "net", &["model-a"]); + let provider = placed_everywhere(test_provider("unrestricted-prov", "net", &["model-a"])); // The provider has an empty siteSelector, so it should generate candidates // for ALL sites in the network when it appears in any overlay. @@ -6361,7 +6423,12 @@ mod tests { let network = test_network("net"); let site_prod = test_site_with_labels("site-prod", "net", &[("env", "prod"), ("region", "us-west")]); let site_staging = test_site_with_labels("site-staging", "net", &[("env", "staging")]); - let provider = test_provider_with_access_policy("prod-only-prov", "net", &["model-a"], &[("env", "prod")]); + let provider = placed_everywhere(test_provider_with_access_policy( + "prod-only-prov", + "net", + &["model-a"], + &[("env", "prod")], + )); // Consumer from prod site should get candidates // Since the provider has an empty siteSelector but restricted access policy, @@ -6482,7 +6549,7 @@ mod tests { let site = test_site_with_labels("known-site", "net", &[("env", "prod")]); let provider_restricted = test_provider_with_access_policy("restricted-prov", "net", &["model-a"], &[("env", "prod")]); - let provider_unrestricted = test_provider("unrestricted-prov", "net", &["model-b"]); + let provider_unrestricted = placed_everywhere(test_provider("unrestricted-prov", "net", &["model-b"])); // Consumer site not in the sites list (unknown identity) let overlay = render_routing_overlay( @@ -6517,15 +6584,19 @@ mod tests { let site_prod = test_site_with_labels("site-prod", "net", &[("env", "prod"), ("team", "platform")]); let site_staging = test_site_with_labels("site-staging", "net", &[("env", "staging"), ("team", "platform")]); - let provider_unrestricted = test_provider("unrestricted-prov", "net", &["model-general"]); - let provider_prod_only = - test_provider_with_access_policy("prod-only-prov", "net", &["model-prod"], &[("env", "prod")]); - let provider_platform_only = test_provider_with_access_policy( + let provider_unrestricted = placed_everywhere(test_provider("unrestricted-prov", "net", &["model-general"])); + let provider_prod_only = placed_everywhere(test_provider_with_access_policy( + "prod-only-prov", + "net", + &["model-prod"], + &[("env", "prod")], + )); + let provider_platform_only = placed_everywhere(test_provider_with_access_policy( "platform-only-prov", "net", &["model-platform"], &[("team", "platform")], - ); + )); // Prod consumer should get all three providers (each provider generates candidates for both sites) let overlay = render_routing_overlay( @@ -6706,7 +6777,7 @@ mod tests { let site = test_site_with_labels("known-site", "net", &[("env", "prod")]); let provider_restricted = test_provider_with_access_policy("local-restricted-prov", "net", &["model-a"], &[("env", "prod")]); - let provider_unrestricted = test_provider("local-unrestricted-prov", "net", &["model-b"]); + let provider_unrestricted = placed_everywhere(test_provider("local-unrestricted-prov", "net", &["model-b"])); // Consumer site not in the sites list (unknown identity) let overlay = render_routing_overlay( diff --git a/operator/src/resources/serving_config.rs b/operator/src/resources/serving_config.rs index 64830931c..60e8d186d 100644 --- a/operator/src/resources/serving_config.rs +++ b/operator/src/resources/serving_config.rs @@ -35,15 +35,24 @@ pub(crate) const MIN_WRITE_INTERVAL: Duration = Duration::from_secs(30); /// Store retention per series, seconds. const WINDOW_SECS: u64 = 60; -/// Freshness window the gateway orders over, milliseconds. -const LOAD_WINDOW_MS: i64 = 30_000; - /// Peer poll interval, the operator's default local scrape interval. const PEER_INTERVAL_MS: u64 = 5_000; /// Peer connect and request timeout, milliseconds. const PEER_TIMEOUT_MS: u64 = 2_000; +/// The freshness window the gateway reads load over, milliseconds: the oldest a sample can +/// be when it arrives (a scrape, a poll, and a poll timeout) plus one poll of slack, so one +/// late poll does not turn a healthy site unknown. 17s at a 5s scrape. +fn load_window_ms(scrape: Duration) -> i64 { + let scrape_ms = u64::try_from(scrape.as_millis()).unwrap_or(u64::MAX); + let window = PEER_INTERVAL_MS + .saturating_add(scrape_ms) + .saturating_add(PEER_TIMEOUT_MS) + .saturating_add(PEER_INTERVAL_MS); + i64::try_from(window).unwrap_or(i64::MAX) +} + /// Gateway-side cap on candidates (`validate_candidates`). const MAX_CANDIDATES: usize = 1024; @@ -81,6 +90,16 @@ pub(crate) struct ServingCandidate { pub(crate) cluster: String, /// Freshness carried from the overlay. pub(crate) fresh: bool, + /// Whether it takes new requests. Omitted when it does, so such a config + /// still loads on a gateway that predates the field. + #[serde(skip_serializing_if = "admits_new")] + pub(crate) admission: AdmissionState, +} + +/// Whether `admission` is the default, which the gateway assumes when absent. +#[expect(clippy::trivially_copy_pass_by_ref, reason = "serde skip_serializing_if signature")] +fn admits_new(admission: &AdmissionState) -> bool { + *admission == AdmissionState::NewAndExisting } /// One peer signals endpoint and the local identity material to reach it. @@ -122,6 +141,8 @@ pub(crate) struct ServingInputs<'input> { pub(crate) local_signals_addr: Option<&'input str>, /// Declared leaf digests per remote site, empty outside pin trust. pub(crate) pins: &'input BTreeMap>, + /// How often operators scrape their providers, taken as every site's. + pub(crate) scrape_interval: Duration, } /// Render from `(site, signals endpoint)` members, `None` when no candidate survives. @@ -133,7 +154,7 @@ pub(crate) fn render<'member, Members>( where Members: IntoIterator, { - let candidates = candidates(&overlay.candidates, &overlay.local_site); + let candidates = candidates(overlay); if candidates.is_empty() { return None; } @@ -155,7 +176,7 @@ where Some(ServingConfig { local_site: overlay.local_site.clone(), window_secs: WINDOW_SECS, - load_window_ms: LOAD_WINDOW_MS, + load_window_ms: load_window_ms(inputs.scrape_interval), candidates, peers, }) @@ -188,51 +209,102 @@ pub(crate) fn dialable_members<'snap>( .collect() } -/// Admitted inference candidates within gateway limits, deduplicated and ordered. -fn candidates(overlay: &[RoutingCandidate], local_site: &str) -> Vec { - // Local first so cold start, before any signal, prefers this site. - let mut unique: BTreeMap<(bool, &str, &str, &str), bool> = BTreeMap::new(); - for candidate in overlay.iter().filter(|candidate| routable(candidate)) { +/// A candidate's identity: remote after local, then site, name, cluster. +type CandidateKey<'overlay> = (bool, &'overlay str, &'overlay str, &'overlay str); + +/// Each routable candidate once, with its freshness and admission. +/// +/// Any stale or more restricted duplicate marks the tuple, whatever the order. +fn dedup<'overlay>( + overlay: impl Iterator, + local_site: &str, +) -> BTreeMap, (bool, AdmissionState)> { + let mut unique = BTreeMap::new(); + for candidate in overlay.filter(|candidate| routable(candidate)) { let key = ( candidate.site != local_site, candidate.site.as_str(), candidate.name.as_str(), candidate.cluster.as_str(), ); - // Any stale duplicate marks the tuple stale, whatever the order. + let admission = candidate.admission_state.unwrap_or(AdmissionState::NewAndExisting); unique .entry(key) - .and_modify(|fresh| *fresh &= candidate.fresh) - .or_insert(candidate.fresh); + .and_modify(|(fresh, held): &mut (bool, AdmissionState)| { + *fresh &= candidate.fresh; + *held = (*held).max(admission); + }) + .or_insert((candidate.fresh, admission)); } - if unique.len() > MAX_CANDIDATES { + unique +} + +/// Inference candidates within gateway limits, deduplicated and ordered. +/// +/// Past the cap, candidates taking new requests are kept before the rest. +fn candidates(overlay: &RoutingOverlay) -> Vec { + // Local first so cold start, before any signal, prefers this site. + let unique = dedup(overlay.candidates.iter().chain(&overlay.excluded), &overlay.local_site); + let mut kept: Vec<_> = unique.into_iter().collect(); + if kept.len() > MAX_CANDIDATES { tracing::warn!( - candidates = unique.len(), + candidates = kept.len(), "serving config: dropping candidates past the gateway cap" ); + kept.sort_by_key(|(key, (_, admission))| (*admission != AdmissionState::NewAndExisting, *key)); + kept.truncate(MAX_CANDIDATES); + kept.sort_by_key(|(key, _)| *key); } - unique - .into_iter() - .take(MAX_CANDIDATES) - .map(|((_, site, name, cluster), fresh)| ServingCandidate { + kept.into_iter() + .map(|((_, site, name, cluster), (fresh, admission))| ServingCandidate { kind: INFERENCE_MODEL, name: name.to_owned(), site: site.to_owned(), cluster: cluster.to_owned(), fresh, + admission, }) .collect() } -/// An inference candidate admitting new requests that the gateway will accept. +/// An inference candidate the gateway will accept, whatever its admission. +/// +/// A candidate not taking new requests stays in, so the gateway can tell a known +/// down model, answered with 503, from an unknown one. Its site must be a DNS-1123 +/// label, as enrolled site names are: the gateway carries it in headers and ids. fn routable(candidate: &RoutingCandidate) -> bool { candidate.kind == INFERENCE_MODEL - && candidate - .admission_state - .is_none_or(|state| state == AdmissionState::NewAndExisting) && [&candidate.name, &candidate.site, &candidate.cluster] .into_iter() .all(|id| valid_id(id)) + && certs::validate_site_name(&candidate.site).is_ok() +} + +/// The candidate sites the serving config refuses for not being DNS-1123 labels. +fn refused_sites(overlay: &RoutingOverlay) -> BTreeSet { + overlay + .candidates + .iter() + .chain(&overlay.excluded) + .filter(|candidate| certs::validate_site_name(&candidate.site).is_err()) + .map(|candidate| candidate.site.clone()) + .collect() +} + +/// The refused sites last seen per `GridNetwork`, keyed by network name. +pub(crate) type RefusedSites = Mutex>>; + +/// Warn when `network`'s set of refused sites changes, so an operator sees why a site never +/// routes. Kept per network, so two networks do not flip each other's warning. +pub(crate) fn warn_refused_sites(overlay: &RoutingOverlay, network: &str, last: &RefusedSites) { + let refused = refused_sites(overlay); + let mut last = last.lock().unwrap_or_else(PoisonError::into_inner); + if last.get(network) != Some(&refused) { + if !refused.is_empty() { + tracing::warn!(network, sites = ?refused, "serving config: refusing candidates whose site is not a DNS-1123 label"); + } + last.insert(network.to_owned(), refused); + } } /// Non-blank and within the gateway's identifier bound. @@ -386,6 +458,7 @@ mod tests { tls_mount: "/etc/praxis/tls", local_signals_addr: None, pins: &NO_PINS, + scrape_interval: Duration::from_secs(5), }; fn cand(name: &str, site: &str, cluster: &str, admission: Option<&str>) -> RoutingCandidate { @@ -403,6 +476,7 @@ mod tests { network: "grid".to_owned(), local_site: "site-a".to_owned(), candidates, + excluded: Vec::new(), selection_policy: None, generated_at: None, } @@ -423,6 +497,21 @@ mod tests { .collect() } + #[test] + fn the_load_window_covers_the_oldest_a_sample_can_arrive_plus_a_poll() { + assert_eq!( + load_window_ms(Duration::from_secs(5)), + 17_000, + "5s poll + 5s scrape + 2s timeout + 5s" + ); + assert_eq!(load_window_ms(Duration::from_secs(1)), 13_000); + let oldest_arrival = PEER_INTERVAL_MS + 5_000 + PEER_TIMEOUT_MS; + assert!( + i64::try_from(oldest_arrival).unwrap_or(i64::MAX) < load_window_ms(Duration::from_secs(5)), + "a sample at the worst-case age is still fresh" + ); + } + #[test] fn renders_the_contract_the_gateway_parses() { let members = [("site-b", "203.0.113.7:9091"), ("site-a", "198.51.100.1:9091")]; @@ -453,18 +542,42 @@ mod tests { } #[test] - fn candidates_the_gateway_would_reject_or_never_admit_are_dropped() { + fn refused_sites_name_only_the_invalid_ones() { + let sites = refused_sites(&overlay(vec![ + cand("llama", "site-a", "pool-a", None), + cand("llama", "site.b", "pool-b", None), + ])); + assert_eq!(sites, BTreeSet::from(["site.b".to_owned()])); + } + + #[test] + fn refused_sites_are_kept_per_network() { + let last = RefusedSites::default(); + let bad = overlay(vec![cand("llama", "site.b", "pool-b", None)]); + let good = overlay(vec![cand("llama", "site-a", "pool-a", None)]); + warn_refused_sites(&bad, "east", &last); + warn_refused_sites(&good, "west", &last); + let held = last.into_inner().unwrap_or_else(PoisonError::into_inner); + assert_eq!(held.get("east"), Some(&BTreeSet::from(["site.b".to_owned()]))); + assert_eq!( + held.get("west"), + Some(&BTreeSet::new()), + "another network does not clear it" + ); + } + + #[test] + fn candidates_the_gateway_would_reject_are_dropped() { let cases = [ - ("excluded", cand("llama", "site-b", "pool-b", Some("none"))), - ( - "existing only", - cand("llama", "site-b", "pool-b", Some("existing_only")), - ), ("blank cluster", cand("llama", "site-b", " ", None)), ( "oversized name", cand(&"m".repeat(MAX_NAME_LEN + 1), "site-b", "pool-b", None), ), + ("site with a dot", cand("llama", "site.b", "pool-b", None)), + ("site with a slash", cand("llama", "site/b", "pool-b", None)), + ("site with a quote", cand("llama", "site\"b", "pool-b", None)), + ("uppercase site", cand("llama", "Site-B", "pool-b", None)), ]; for (label, bad) in cases { assert!(render(&overlay(vec![bad]), [], &INPUTS).is_none(), "{label}"); @@ -474,6 +587,68 @@ mod tests { assert!(render(&overlay(vec![mcp]), [], &INPUTS).is_none(), "mcp_tool"); } + #[test] + fn an_excluded_candidate_reaches_the_serving_config_but_not_the_overlay_wire() { + let mut source = overlay(vec![cand("llama", "site-a", "pool-a", None)]); + source.excluded = vec![cand("llama", "site-b", "pool-b", Some("none"))]; + let rendered = render(&source, [], &INPUTS).expect("rendered"); + let config: serde_json::Value = serde_json::from_str(&to_text(&rendered).expect("text")).expect("json"); + let pool_b = config["candidates"] + .as_array() + .expect("candidates") + .iter() + .find(|c| c["cluster"] == "pool-b") + .expect("the excluded candidate is listed"); + assert_eq!(pool_b["admission"], "none"); + let wire = serde_json::to_string(&source).expect("overlay json"); + assert!(!wire.contains("pool-b"), "the overlay wire is unchanged: {wire}"); + } + + #[test] + fn past_the_cap_candidates_taking_new_requests_are_kept_first() { + let excluded: Vec<_> = (0..MAX_CANDIDATES) + .map(|i| cand("llama", "site-a", &format!("down-{i:04}"), Some("none"))) + .collect(); + let mut source = overlay(vec![cand("llama", "site-z", "up", None)]); + source.excluded = excluded; + let rendered = render(&source, [], &INPUTS).expect("rendered"); + assert_eq!(rendered.candidates.len(), MAX_CANDIDATES); + assert!( + rendered.candidates.iter().any(|c| c.cluster == "up"), + "the one admitted candidate survives the cap" + ); + } + + #[test] + fn a_candidate_not_taking_new_requests_stays_in_with_its_admission() { + let rendered = render( + &overlay(vec![ + cand("llama", "site-a", "pool-a", None), + cand("llama", "site-b", "pool-b", Some("none")), + cand("llama", "site-d", "pool-d", Some("existing_only")), + ]), + [], + &INPUTS, + ) + .expect("rendered"); + let config: serde_json::Value = serde_json::from_str(&to_text(&rendered).expect("text")).expect("json"); + let admission: Vec<(&str, Option<&str>)> = config["candidates"] + .as_array() + .expect("candidates") + .iter() + .map(|c| (c["site"].as_str().unwrap_or(""), c["admission"].as_str())) + .collect(); + assert_eq!( + admission, + [ + ("site-a", None), + ("site-b", Some("none")), + ("site-d", Some("existing_only")) + ], + "the default is omitted, so an older gateway still loads an all-admitted config" + ); + } + #[test] fn unsafe_or_irrelevant_peers_are_refused() { let cases = [ @@ -500,10 +675,9 @@ mod tests { let other = render(&two_site(), [("site-z", "203.0.113.9:9091")], &INPUTS).expect("routable"); assert!(other.peers.is_empty(), "a site with no candidate is not polled"); let bad_name = overlay(vec![cand("llama", "Site_B", "pool-b", None)]); - let config = render(&bad_name, [("Site_B", "203.0.113.7:9091")], &INPUTS).expect("routable"); assert!( - config.peers.is_empty(), - "a non-DNS site name would fail the gateway's SNI" + render(&bad_name, [("Site_B", "203.0.113.7:9091")], &INPUTS).is_none(), + "a non-DNS site name is neither a candidate nor a peer" ); } diff --git a/operator/src/signals.rs b/operator/src/signals.rs index 63ffc569f..406bbae93 100644 --- a/operator/src/signals.rs +++ b/operator/src/signals.rs @@ -67,6 +67,18 @@ pub struct Observation { /// somebody else's scrape and the caller cannot repair it. #[must_use] pub fn parse(text: &str) -> Vec { + parse_scrape(text) + .into_iter() + .filter_map(|(o, republishable)| republishable.then_some(o)) + .collect() +} + +/// Parse a scrape this site made itself: every finite sample, each paired with whether +/// [`parse`] would keep it for republishing. +/// +/// Counters and histogram parts stay, for this site's own latency and error windows. +#[must_use] +pub(crate) fn parse_scrape(text: &str) -> Vec<(Observation, bool)> { // Types first: a declaration may follow its samples, and reading in one // pass would admit a counter that had not been typed yet. let types: HashMap<&str, &str> = text @@ -81,13 +93,14 @@ pub fn parse(text: &str) -> Vec { text.lines() .filter(|l| !l.trim_start().starts_with('#')) .filter_map(parse_sample) - .filter(|o| { + .filter(|o| o.value.is_finite()) + .map(|o| { // Gauges and untyped only: a relayed counter reports our restarts, // and an aggregate we did not observe cannot be recombined. The name // check catches aggregates that arrive untyped. - matches!(types.get(o.metric.as_str()), None | Some(&("gauge" | "untyped"))) - && !is_aggregate_part(&o.metric) - && o.value.is_finite() + let republishable = matches!(types.get(o.metric.as_str()), None | Some(&("gauge" | "untyped"))) + && !is_aggregate_part(&o.metric); + (o, republishable) }) .collect() } @@ -479,6 +492,20 @@ impl SignalStore { (out, oldest) } + /// Every sample still served, across targets. + #[must_use] + pub fn current(&self) -> Vec { + let Ok(guard) = self.inner.read() else { + return Vec::new(); + }; + let now = Instant::now(); + guard + .values() + .filter(|held| held.expires_at > now) + .flat_map(|held| held.samples.iter().cloned()) + .collect() + } + /// Targets currently held and unexpired. #[must_use] pub fn targets(&self) -> Vec { @@ -815,6 +842,7 @@ fn classify(error: &MetricsScrapeError) -> PollOutcome { } }, MetricsScrapeError::Encoding(_) => PollOutcome::Encoding, + MetricsScrapeError::BodyTooLarge(_) => PollOutcome::Transport, MetricsScrapeError::InvalidUrl(_) | MetricsScrapeError::HttpWithTls(_) | MetricsScrapeError::TlsMaterial(_) @@ -1105,7 +1133,7 @@ impl PollPeers { .map(|(peer, url, pins)| async move { let (body, date) = self.poll_one(&peer, &url, &pins).await?; let now_ms = wall_millis(SystemTime::now(), Duration::ZERO); - let mut observations = retain_origin(parse(&body), &peer); + let mut observations = bound_peer(retain_origin(parse(&body), &peer), &peer); reexpress_peer_ages(&mut observations, date, now_ms); Some((peer, observations)) }); @@ -1207,6 +1235,86 @@ fn with_query(url: &str, query: &str) -> Option { http::Uri::from_parts(parts).ok().map(|uri| uri.to_string()) } +/// Names a hub keeps from a peer: the cross-site contract, plus the EPP pool averages and +/// ready counts the gateway routes on until it routes on saturation. A peer's custom +/// `signalNames` are dropped here. +const PEER_SIGNAL_NAMES: [&str; 16] = [ + crate::readiness::READY_SIGNAL, + crate::readiness::IN_FLIGHT_SIGNAL, + crate::readiness::CAPACITY_SIGNAL, + crate::readiness::SATURATION_SIGNAL, + crate::latency::TTFT_P50_SIGNAL, + crate::latency::TTFT_P90_SIGNAL, + crate::latency::TPOT_SIGNAL, + crate::latency::PREFILL_SIGNAL, + crate::latency::ERROR_RATIO_SIGNAL, + "inference_pool_average_queue_size", + "llm_d_epp_average_queue_size", + "inference_pool_average_running_requests", + "llm_d_epp_average_running_requests", + "inference_pool_average_kv_cache_utilization", + "llm_d_epp_average_kv_cache_utilization", + "llm_d_epp_ready_endpoints", +]; + +/// Most providers a hub keeps from one peer, so a peer cannot grow the hub's series without +/// bound. The first in name order are kept. +pub const MAX_PEER_PROVIDERS: usize = 64; + +/// Signals that are shares, which a peer may not report above one. +const UNIT_SIGNALS: [&str; 2] = [crate::readiness::READY_SIGNAL, crate::latency::ERROR_RATIO_SIGNAL]; + +/// A value a hub accepts: finite and non-negative, and at most one for a share. +fn plausible(observation: &Observation) -> bool { + observation.value.is_finite() + && observation.value >= 0.0 + && (observation.value <= 1.0 || !UNIT_SIGNALS.contains(&observation.metric.as_str())) +} + +/// Why a hub refuses `observation` from a peer, `None` when it accepts it. +fn refusal(observation: &Observation) -> Option<&'static str> { + if !PEER_SIGNAL_NAMES.contains(&observation.metric.as_str()) { + Some("name") + } else if observation + .labels + .get(PROVIDER_LABEL) + .is_none_or(|provider| certs::validate_site_name(provider).is_err()) + { + Some("provider") + } else if !plausible(observation) { + Some("value") + } else { + None + } +} + +/// Keep what a hub accepts from `peer`: an allowed name, a `grid_provider` that is a DNS-1123 +/// label, a plausible value, and at most [`MAX_PEER_PROVIDERS`] providers. Each refusal is +/// counted by reason. +fn bound_peer(observations: Vec, peer: &str) -> Vec { + let mut kept = Vec::with_capacity(observations.len()); + for observation in observations { + match refusal(&observation) { + Some(reason) => crate::metrics::record_peer_signal_refused(peer, reason), + None => kept.push(observation), + } + } + let providers: std::collections::BTreeSet<&str> = kept + .iter() + .filter_map(|o| o.labels.get(PROVIDER_LABEL).map(String::as_str)) + .collect(); + let Some(&last) = providers.iter().nth(MAX_PEER_PROVIDERS.saturating_sub(1)) else { + return kept; + }; + let last = last.to_owned(); + let before = kept.len(); + kept.retain(|o| o.labels.get(PROVIDER_LABEL).is_some_and(|p| *p <= last)); + for _ in kept.len()..before { + crate::metrics::record_peer_signal_refused(peer, "provider_cap"); + } + kept +} + /// Keep only the observations a peer made itself. /// /// A relayed copy is dropped rather than trusted, so every site's data reaches @@ -1258,6 +1366,88 @@ fn reexpress_peer_ages(observations: &mut [Observation], date: Option Observation { + Observation { + metric: metric.to_owned(), + labels: BTreeMap::from([ + (SITE_LABEL.to_owned(), "retail".to_owned()), + (PROVIDER_LABEL.to_owned(), provider.to_owned()), + ]), + value, + timestamp_ms: None, + } + } + + #[test] + #[expect(clippy::too_many_lines, reason = "one table of accepted and refused samples")] + fn a_hub_refuses_peer_names_providers_and_values_outside_the_contract() { + let cases = [ + ( + "contract name", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", 0.5), + true, + ), + ( + "gateway load name", + peer_sample("inference_pool_average_queue_size", "pool", 3.0), + true, + ), + ("custom name", peer_sample("my_custom_queue", "pool", 3.0), false), + ( + "provider not a label", + peer_sample(crate::readiness::READY_SIGNAL, "Pool_1", 1.0), + false, + ), + ( + "not finite", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", f64::NAN), + false, + ), + ( + "infinite", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", f64::INFINITY), + false, + ), + ( + "negative", + peer_sample(crate::readiness::IN_FLIGHT_SIGNAL, "pool", -1.0), + false, + ), + ( + "share above one", + peer_sample(crate::latency::ERROR_RATIO_SIGNAL, "pool", 1.5), + false, + ), + ( + "saturation above one", + peer_sample(crate::readiness::SATURATION_SIGNAL, "pool", 1.5), + true, + ), + ]; + for (label, observation, kept) in cases { + assert_eq!( + bound_peer(vec![observation], "retail").len(), + usize::from(kept), + "{label}" + ); + } + } + + #[test] + fn a_hub_keeps_at_most_the_first_providers_of_a_peer_in_name_order() { + let observations: Vec = (0..MAX_PEER_PROVIDERS + 5) + .map(|i| peer_sample(crate::readiness::READY_SIGNAL, &format!("p{i:03}"), 1.0)) + .collect(); + let kept = bound_peer(observations, "retail"); + assert_eq!(kept.len(), MAX_PEER_PROVIDERS); + assert!( + kept.iter() + .all(|o| o.labels.get(PROVIDER_LABEL).is_some_and(|p| p.as_str() < "p064")), + "the first {MAX_PEER_PROVIDERS} in name order are kept" + ); + } + #[test] fn peers_are_dialed_at_a_dialable_signals_endpoint() { let member = |site: &str, endpoint: &str, signals: Option<&str>| MemberRecord { @@ -1302,7 +1492,7 @@ mod tests { let mut request = [0_u8; 1024]; drop(stream.read(&mut request).await); drop(release.await); - let body = format!("load{{grid_site=\"{site}\"}} 1\n"); + let body = format!("inference_pool_average_queue_size{{grid_site=\"{site}\",grid_provider=\"pool\"}} 1\n"); let response = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}", body.len()); drop(stream.write_all(response.as_bytes()).await); }); diff --git a/signals/src/signals.rs b/signals/src/signals.rs index 1590913a8..d156ace6e 100644 --- a/signals/src/signals.rs +++ b/signals/src/signals.rs @@ -154,17 +154,26 @@ impl LoadStore { format!("{site}/{cluster}").into_boxed_str() } - /// Most recent sample of `metric` for `key`. Test-only since scoring reads - /// [`Self::window_worst`]. - #[cfg(test)] + /// Most recent sample of `metric` for `key`. + #[must_use] pub fn latest(&self, key: &str, metric: &str) -> Option { let provider = self.providers.get(key)?; provider.metrics.get(metric)?.samples.last().copied() } + /// When anything was last observed for `key`, across all its metrics. + #[must_use] + pub fn newest_at(&self, key: &str) -> Option { + let provider = self.providers.get(key)?; + provider + .metrics + .values() + .filter_map(|series| series.samples.last().map(|sample| sample.at_ms)) + .max() + } + /// Most recent sample of `metric` for `key` younger than `max_age_ms`. - /// Test-only since scoring reads [`Self::window_worst`]. - #[cfg(test)] + #[must_use] pub fn fresh(&self, key: &str, metric: &str, now_ms: i64, max_age_ms: i64) -> Option { // Range starts at zero: a future timestamp (publisher clock ahead) yields // a negative age that would otherwise read as fresh forever. diff --git a/tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml b/tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml new file mode 100644 index 000000000..e89bbe1b1 --- /dev/null +++ b/tests/e2e/topologies/grid-single-cluster-multi-gateway/.grid-single-cluster-multi-gateway-3286232.resolved.yaml @@ -0,0 +1,530 @@ +apiVersion: forge.praxis.dev/v1alpha1 +kind: Environment +metadata: + name: grid-single-cluster-multi-gateway +spec: + runtime: + provider: docker + clusterPrefix: grid-single-cluster-multi-gateway + network: + crossCluster: true + dnsZone: grid-single-cluster-multi-gateway.test + clusters: + - name: single + stacks: + - metallb + - tls-bootstrap + - provider-a-operator-base + - vcr-backend + - provider-a-site + - provider-gateway-a + - provider-gateway-b + - provider-gateway-c + - consumer-gateway-a + - consumer-gateway-b + properties: + region: single + role: combined + siteName: single + gatewayImage: registry.example/ai:run-4 + operatorImage: registry.example/operator:run-4 + vcrImage: registry.example/vcr:run-4 + imagePullPolicy: Never + gatewayImageRepo: registry.example/ai + gatewayImageTag: run-4 + operatorImageRepo: registry.example/operator + operatorImageTag: run-4 + overlaySyncImage: registry.example/sync:run-4 + overlaySyncImageRepo: registry.example/sync + overlaySyncImageTag: run-4 + stacks: + tls-bootstrap: + description: Ephemeral same-CA identities for this isolated qualification + steps: + - type: manifest + path: resources/common/grid-system-namespace.yaml + - type: exec + command: + - bash + - -c + - set -eu; d=$(mktemp -d); trap 'rm -rf "${d}"' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -days 1 -subj "/O=ai-grid/CN=qualification-ca" -keyout "${d}/ca.key" -out "${d}/ca.crt" >/dev/null 2>&1; openssl req -newkey rsa:2048 -nodes -subj "/O=ai-grid/CN=grid-gateway.grid.internal" -keyout "${d}/gateway.key" -out "${d}/gateway.csr" >/dev/null 2>&1; printf "subjectAltName=DNS:grid-gateway.grid.internal,DNS:provider-a.grid.internal,DNS:provider-b.grid.internal,DNS:provider-c.grid.internal,DNS:grid-system.svc.cluster.local" > "${d}/ext.cnf"; openssl x509 -req -days 1 -in "${d}/gateway.csr" -CA "${d}/ca.crt" -CAkey "${d}/ca.key" -CAcreateserial -out "${d}/gateway.crt" -extfile "${d}/ext.cnf" >/dev/null 2>&1; kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create secret generic consumer-gateway-tls --from-file=ca.crt="${d}/ca.crt" --from-file=tls.crt="${d}/gateway.crt" --from-file=tls.key="${d}/gateway.key" --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - >/dev/null; kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create secret generic vcr-inference-credential --from-literal=token="$(openssl rand -hex 32)" --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - >/dev/null + metallb: + description: MetalLB load balancer with auto-configured address pool + steps: + - type: url + url: https://raw.githubusercontent.com/metallb/metallb/v0.14.9/config/manifests/metallb-native.yaml + sha256: 951065e85692aa106f1bb5d5a487d9306154923a794ab1d82122881cbaf588e4 + - type: wait + resource: deployment/controller + namespace: metallb-system + condition: available + timeout: 120s + - type: metallb-auto-pool + name: forge-pool + provider-a-operator-base: + description: Single Grid operator for the shared Kind control plane + steps: + - type: helm + release: grid-operator + chart: charts/grid-operator + version: 0.1.0 + namespace: grid-system + values: + image: + repository: '{{ cluster.properties.operatorImageRepo }}' + tag: '{{ cluster.properties.operatorImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + swim: + siteName: single + seeds: '' + service: + enabled: true + type: LoadBalancer + gateway: + serviceName: provider-gateway-a + port: '8443' + - type: wait + resource: deployment/grid-operator + namespace: grid-system + condition: available + timeout: 120s + - type: helm + release: grid-operator + chart: charts/grid-operator + version: 0.1.0 + namespace: grid-system + values: + image: + repository: '{{ cluster.properties.operatorImageRepo }}' + tag: '{{ cluster.properties.operatorImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + swim: + siteName: single + seeds: '' + advertiseAddress: 172.18.255.231:7946 + service: + enabled: true + type: LoadBalancer + gateway: + serviceName: provider-gateway-a + port: '8443' + vcr-backend: + description: vllm-vcr inference backend for demo scenarios + steps: + - type: manifest + path: resources/common/grid-system-namespace.yaml + - type: template-manifest + path: resources/common/vcr-provider-a.yaml + - type: template-manifest + path: resources/common/vcr-provider-b.yaml + - type: template-manifest + path: resources/common/vcr-provider-c.yaml + - type: manifest + path: resources/common/backend-network-policy.yaml + - type: manifest + path: resources/common/qualification-client.yaml + - type: wait + resource: deployment/vcr-inference-provider-a + namespace: grid-system + condition: available + timeout: 120s + provider-a-site: + description: Single GridSite with Grid CRs via grid-site chart + steps: + - type: helm + release: grid-site-a + chart: charts/grid-site + version: 0.1.0 + namespace: grid-system + values: + commonLabels: + grid.praxis.fast/auto-discover-sites: 'true' + gridNetwork: + name: grid-single-cluster-multi-gateway + gridId: grid-single-cluster-multi-gateway-v1 + region: provider-a + zone: provider-a-1 + routingPolicy: scoreFirst + scoringPolicy: + strategy: noMetrics + selectionPolicy: + mode: roundRobin + swim: + probeInterval: 5s + suspicionTimeout: 15s + gossipNodes: 3 + tls: + caSecretRef: + name: consumer-gateway-tls + namespace: grid-system + siteSecretRef: + name: consumer-gateway-tls + namespace: grid-system + gatewayRefs: + - name: consumer-gateway-a + namespace: grid-system + localSiteName: single + - name: consumer-gateway-b + namespace: grid-system + localSiteName: single + gridSite: + name: single + region: single + zone: single-1 + providerSiteLabel: single + inferenceProviders: + - name: vcr-provider-a-provider + gridNetworkRef: grid-single-cluster-multi-gateway + gatewayRef: provider-gateway-a + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-provider-a.grid-system.svc.cluster.local:8000 + siteSelector: + matchLabels: + grid.praxis.fast/provider-site: single + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: 10s + timeout: 5s + - name: vcr-provider-b-provider + gridNetworkRef: grid-single-cluster-multi-gateway + gatewayRef: provider-gateway-a + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-provider-b.grid-system.svc.cluster.local:8000 + siteSelector: + matchLabels: {} + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: 10s + timeout: 5s + - name: vcr-provider-c-provider + gridNetworkRef: grid-single-cluster-multi-gateway + gatewayRef: provider-gateway-b + providerKind: vllm-vcr + backendKind: local + endpoint: http://vcr-inference-provider-c.grid-system.svc.cluster.local:8000 + siteSelector: + matchLabels: {} + accessPolicy: + siteSelector: + matchLabels: {} + models: + - name: Qwen/Qwen3-0.6B + capabilities: + - text_generation + contextWindow: 4096 + healthCheck: + path: /health + interval: 10s + timeout: 5s + provider-gateway-a: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: template-file + source: configs/provider/praxis-a.yaml + target: .forge/runtime/{{ cluster.name }}/provider-a-praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create configmap provider-gateway-a-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider-a-praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - + - type: helm + release: provider-gateway-a + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: provider-gateway-a + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: provider-gateway-a-config + port: + containerPort: 8443 + name: https-mtls + service: + type: LoadBalancer + port: 8443 + health: + readiness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: consumer-gateway-tls + credentials: + - name: vcr-inference-credential + mountPath: /etc/praxis/credentials/vcr-inference + podLabels: + grid.praxis.fast/backend-access: provider-gateway + grid.praxis.fast/provider-site: '{{ cluster.name }}' + - type: wait + resource: deployment/provider-gateway-a + namespace: grid-system + condition: available + timeout: 120s + provider-gateway-b: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: template-file + source: configs/provider/praxis-b.yaml + target: .forge/runtime/{{ cluster.name }}/provider-b-praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create configmap provider-gateway-b-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider-b-praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - + - type: helm + release: provider-gateway-b + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: provider-gateway-b + replicaCount: 1 + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: provider-gateway-b-config + port: + containerPort: 8443 + name: https-mtls + service: + type: LoadBalancer + port: 8443 + health: + readiness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: consumer-gateway-tls + credentials: + - name: vcr-inference-credential + mountPath: /etc/praxis/credentials/vcr-inference + podLabels: + grid.praxis.fast/backend-access: provider-gateway + grid.praxis.fast/provider-site: '{{ cluster.name }}' + - type: wait + resource: deployment/provider-gateway-b + namespace: grid-system + condition: available + timeout: 120s + provider-gateway-c: + description: Praxis provider gateway with mTLS and credential mounts + steps: + - type: template-file + source: configs/provider/praxis-c.yaml + target: .forge/runtime/{{ cluster.name }}/provider-c-praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-single -n grid-system create configmap provider-gateway-c-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/provider-c-praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-single apply -f - + - type: helm + release: provider-gateway-c + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: provider-gateway-c + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: provider-gateway-c-config + port: + containerPort: 8443 + name: https-mtls + service: + type: LoadBalancer + port: 8443 + health: + readiness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 3 + periodSeconds: 5 + liveness: + tcpSocket: + port: https-mtls + initialDelaySeconds: 5 + periodSeconds: 10 + tls: + enabled: true + existingSecret: consumer-gateway-tls + credentials: + - name: vcr-inference-credential + mountPath: /etc/praxis/credentials/vcr-inference + podLabels: + grid.praxis.fast/backend-access: provider-gateway + grid.praxis.fast/provider-site: '{{ cluster.name }}' + - type: wait + resource: deployment/provider-gateway-c + namespace: grid-system + condition: available + timeout: 120s + consumer-gateway-a: + description: Praxis consumer gateway with operator-managed overlay + steps: + - type: template-file + source: configs/consumer/praxis-a.yaml + target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} -n grid-system create configmap consumer-gateway-a-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway-a + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: consumer-gateway-a + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: consumer-gateway-a-config + service: + type: ClusterIP + overlay: + enabled: true + existingConfigMap: grid-overlay-grid-single-cluster--consumer-gateway-a-db750bac + tls: + enabled: true + existingSecret: consumer-gateway-tls + podLabels: + grid.praxis.fast/consumer-site: '{{ cluster.name }}' + - type: wait + resource: deployment/consumer-gateway-a + namespace: grid-system + condition: available + timeout: 120s + consumer-gateway-b: + description: Praxis consumer gateway with operator-managed overlay + steps: + - type: template-file + source: configs/consumer/praxis-b.yaml + target: .forge/runtime/{{ cluster.name }}/consumer/praxis.yaml + - type: exec + command: + - bash + - -c + - kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} -n grid-system create configmap consumer-gateway-b-config --from-file=praxis.yaml=.forge/runtime/{{ cluster.name }}/consumer/praxis.yaml --dry-run=client -o yaml | kubectl --context kind-grid-single-cluster-multi-gateway-{{ cluster.name }} apply -f - + - type: helm + release: consumer-gateway-b + chart: charts/praxis-gateway + version: 0.1.0 + namespace: grid-system + values: + fullnameOverride: consumer-gateway-b + image: + repository: '{{ cluster.properties.gatewayImageRepo }}' + tag: '{{ cluster.properties.gatewayImageTag }}' + pullPolicy: '{{ cluster.properties.imagePullPolicy }}' + podSecurityContext: + runAsUser: 100 + runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi + config: + existingConfigMap: consumer-gateway-b-config + service: + type: ClusterIP + overlay: + enabled: true + existingConfigMap: grid-overlay-grid-single-cluster--consumer-gateway-b-de751065 + tls: + enabled: true + existingSecret: consumer-gateway-tls + podLabels: + grid.praxis.fast/consumer-site: '{{ cluster.name }}' + - type: wait + resource: deployment/consumer-gateway-b + namespace: grid-system + condition: available + timeout: 120s