diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml index a0072e49..7fcf9af4 100644 --- a/.github/workflows/ci-external-config.yaml +++ b/.github/workflows/ci-external-config.yaml @@ -34,7 +34,7 @@ jobs: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false - name: Install KEDA Autoscaler run: | diff --git a/.github/workflows/ci-full.yaml b/.github/workflows/ci-full.yaml index 9416fc7d..6811db2a 100644 --- a/.github/workflows/ci-full.yaml +++ b/.github/workflows/ci-full.yaml @@ -34,7 +34,7 @@ jobs: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false - name: Install KEDA Autoscaler run: | diff --git a/.github/workflows/ci-local.sh b/.github/workflows/ci-local.sh index 7f255758..6bc72588 100644 --- a/.github/workflows/ci-local.sh +++ b/.github/workflows/ci-local.sh @@ -53,6 +53,7 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c helm repo update kubectl create namespace monitoring helm install prometheus-operator prometheus-community/kube-prometheus-stack \ + --version 89.2.4 \ --namespace monitoring \ --set prometheusOperator.createCustomResource=false \ --set defaultRules.create=false \ diff --git a/README.md b/README.md index 45a9fe60..6d77631c 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Currently, SuperSONIC supports the following functionality: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false ``` - [KEDA](https://keda.sh) CRDs (only if using autoscaling) diff --git a/docs/.values-table.md b/docs/.values-table.md index ad7c7e1e..cdf1a510 100644 --- a/docs/.values-table.md +++ b/docs/.values-table.md @@ -3,8 +3,10 @@ | Key | Type | Default | Description | |-----|------|---------|-------------| | nameOverride | string | `""` | Unique identifier of SuperSONIC instance (equal to release name by default) | -| serverLoadMetric | string | `""` | A metric used by both KEDA autoscaler and Envoy's prometheus-based rate limiter. # Default metric (inference queue latency) is defined in templates/_helpers.tpl | -| serverLoadThreshold | int | `100` | Threshold for the metric | +| serverLoadMetric | string | `""` | Prometheus query used by the KEDA autoscaler and (per healthy replica) by Envoy's prometheus-based rate limiter. Leave empty for the default metric: requests in flight between Envoy and Triton divided by requests being executed, i.e. how many replicas the current load needs. Reads ~1 with no queueing, grows linearly with overload, and has no model-specific constants. See templates/_helpers/_scaling-metric.tpl and the configuration guide. A custom query set here is used verbatim by both consumers. | +| serverLoadThreshold | int | `2` | Threshold for the scaling metric. KEDA scales to ceil(metric / threshold): 2 targets "waiting = serving" per replica; 1.5 trades GPUs for latency. | +| serverLoadRateInterval | string | `"1m"` | Range-vector window used by rate() in the default metric. Keep it at or above 4x your Prometheus scrape interval. | +| serverAdmissionThreshold | int | `3` | Threshold for Envoy's prometheus-based rate limiter (scaling metric per healthy replica). Kept above serverLoadThreshold so new clients are rejected only when scaling can no longer keep up. | | scaleFromZero | object | `{"admissionImage":"python:3.14-slim","enabled":false,"holdMinReplicasSeconds":300,"readyTimeoutSeconds":300}` | On RepositoryIndex, scale Triton to at least max(1, keda.minReplicaCount) replicas and return the index only after Envoy has a healthy Triton upstream. Requires keda.enabled and envoy.enabled. | | scaleFromZero.enabled | bool | `false` | Enable scale from zero | | scaleFromZero.readyTimeoutSeconds | int | `300` | Seconds to wait for a healthy Triton upstream before rejecting RepositoryIndex | @@ -60,6 +62,7 @@ | envoy.auth.url | string | `""` | | | envoy.auth.port | int | `443` | | | keda.enabled | bool | `false` | Enable autoscaling (requires Prometheus to also be enabled). Autoscaling will be based on the metric from parameter ``serverLoadMetric``; new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. | +| keda.metricType | string | `"AverageValue"` | How the HPA interprets the scaling metric. AverageValue (the default) treats it as fleet-wide "replicas needed": desired = ceil(metric / threshold). Set to Value for a custom per-replica metric: desired = ceil(current_replicas * metric / threshold). | | keda.minReplicaCount | int | `1` | Minimum and maximum number of Triton servers. Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled). With scaleFromZero, a RepositoryIndex request scales Triton to max(1, minReplicaCount), and upgrades keep the live ScaledObject minReplicaCount. | | keda.maxReplicaCount | int | `2` | | | keda.pollingInterval | int | `30` | How often KEDA polls Prometheus | diff --git a/docs/configuration-guide.rst b/docs/configuration-guide.rst index 8fe2d7fe..a92128a9 100644 --- a/docs/configuration-guide.rst +++ b/docs/configuration-guide.rst @@ -282,20 +282,60 @@ Prometheus is needed to scrape metrics for monitoring, as well as for the rate l 8. (Optional) Configure Metrics for Scaling and Rate Limiting =============================================================== -Both the rate limiter and the autoscaler are currently configured to use the same Prometheus metric and threshold. -They are defined in the ``serverLoadMetric`` and ``serverLoadThreshold`` parameters at the root level of the values file. -The default metric is the inference queue time at the Triton servers, as defined in -`here `_. - -When the metric value exceeds the threshold, the following happens: - -- Autoscaler scales up the number of Triton servers if possible. -- Envoy proxy rejects new ``RepositoryIndex`` requests. - -The pre-configured Grafana dashboard contains a graph of this metric, entitled "Server Load Metric". -The Prometheus query for the graph is automatically inferred from the value of ``serverLoadMetric`` parameter. -The graph also displays the threshold value defined in ``serverLoadThreshold`` parameter. - +The autoscaler and the Prometheus-based rate limiter are driven by one Prometheus +query, set by ``serverLoadMetric`` at the root of the values file and rendered in +``templates/_helpers/_scaling-metric.tpl``. + +The default metric +------------------- + +By default, SuperSONIC estimates **how many Triton replicas the current in-flight +work needs**:: + + R_needed = L_envoy / max(L_service / R_healthy, 1) + +Each input is measured — the ``rate()`` of a cumulative time counter equals the +mean number of requests inside that stage: + +- ``L_envoy`` — requests in flight between Envoy and Triton (queued, executing, + or on the wire): ``sum(rate(envoy_cluster_upstream_rq_time_sum{...}[1m])) / 1e3``. +- ``L_service`` — requests being executed across all models and pods: + ``sum(rate(nv_inference_request_duration_us − nv_inference_queue_duration_us)) / 1e6``. + Models are weighted by the time they consume, so the metric has no + model-specific constants and one threshold works for any mixture. +- ``R_healthy`` — Triton endpoints Envoy routes to: + ``max(envoy_cluster_membership_healthy{...})``. + +In PromQL, ``clamp_min(v, s)`` spells ``max(v, s)``. The floors encode that a +healthy replica can always execute at least one request (which lets an +underutilized fleet scale down), and that at zero replicas the metric reads +"requests in flight" instead of dividing by zero. + +Per healthy replica, the metric is the inflation clients experience: 1 means +every in-flight request is being executed, 2 means requests wait as long as they +are served, and it keeps growing linearly with overload. + +Thresholds +----------- + +- ``serverLoadThreshold`` (default ``2``) — KEDA scales to + ``ceil(metric / threshold)`` (``metricType: AverageValue``). ``2`` targets + "waiting ≈ serving"; ``1.5`` trades GPUs for latency. The unloaded floor is + ~1.2–1.3 (network transit), so values below that over-provision. +- ``serverAdmissionThreshold`` (default ``3``) — Envoy rejects new + ``RepositoryIndex`` requests when the per-replica metric exceeds it. Kept + above ``serverLoadThreshold``: the autoscaler settles near its threshold, and + gating admission there would reject clients during normal operation. +- ``serverLoadRateInterval`` (default ``1m``) — the ``rate()`` window; keep it + at or above 4x the Prometheus scrape interval. + +Custom metrics +--------------- + +If ``serverLoadMetric`` is set, it is used **verbatim** by both consumers: KEDA +compares it against ``serverLoadThreshold`` and the rate limiter against +``serverAdmissionThreshold``. Set ``keda.metricType: Value`` for per-replica +custom metrics. 9. (Optional) Deploy Grafana Dashboard ========================================== diff --git a/helm/supersonic/README.md b/helm/supersonic/README.md index 45a9fe60..6d77631c 100644 --- a/helm/supersonic/README.md +++ b/helm/supersonic/README.md @@ -55,7 +55,7 @@ Currently, SuperSONIC supports the following functionality: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false ``` - [KEDA](https://keda.sh) CRDs (only if using autoscaling) diff --git a/helm/supersonic/templates/NOTES.txt b/helm/supersonic/templates/NOTES.txt index dfa0b48d..d4495658 100644 --- a/helm/supersonic/templates/NOTES.txt +++ b/helm/supersonic/templates/NOTES.txt @@ -24,7 +24,9 @@ SuperSONIC chart successfully installed! Scaling metric:{{ if not ( eq .Values.serverLoadMetric "" ) }} {{ .Values.serverLoadMetric }}{{ else }}{{ include "supersonic.defaultMetric" . | nindent 4 }}{{ end }} -Scaling threshold: {{ include "supersonic.defaultThreshold" . }} +Scaling threshold (KEDA): {{ include "supersonic.defaultThreshold" . }} + +Admission threshold (Envoy rate limiter): {{ include "supersonic.admissionThreshold" . }} ┌-----------------------------------------------------------------------------┐ | Documentation: https://fastmachinelearning.org/SuperSONIC diff --git a/helm/supersonic/templates/_helpers/_scaling-metric.tpl b/helm/supersonic/templates/_helpers/_scaling-metric.tpl index a6d3e7bd..083f659a 100644 --- a/helm/supersonic/templates/_helpers/_scaling-metric.tpl +++ b/helm/supersonic/templates/_helpers/_scaling-metric.tpl @@ -1,23 +1,93 @@ {{/* -Get default scaling metric +Scaling and admission metrics. + +The default scaling metric estimates how many replicas the current in-flight +work needs: + + R_needed = L_envoy / max(L_service / R_healthy, 1) + + L_envoy - mean requests in flight between Envoy and Triton: rate of + Envoy's cumulative upstream request-time counter (ms -> /1e3). + L_service - mean requests being executed across all models and pods: rate + of Triton's request-duration minus queue-duration counters + (us -> /1e6). + R_healthy - Triton endpoints Envoy routes to (max across Envoy pods). + +clamp_min(v, s) is PromQL for max(v, s). The floors encode that a healthy +replica can always execute at least one request, and that at zero replicas +the metric reads "requests in flight" instead of dividing by zero. + +KEDA consumes "supersonic.defaultMetric" (the form above) with metricType +AverageValue: desired = ceil(R_needed / serverLoadThreshold). The Envoy Lua +filter consumes "supersonic.admissionMetric" (the same per healthy replica) +and rejects RepositoryIndex above "supersonic.admissionThreshold". + +If .Values.serverLoadMetric is set, both helpers return it verbatim. +*/}} + +{{/* +Range-vector window for rate() in the default metric. +Keep it at >= 4x the Prometheus scrape interval. +*/}} +{{- define "supersonic.rateInterval" -}} +{{- default "1m" .Values.serverLoadRateInterval -}} +{{- end -}} + +{{/* +Healthy Triton endpoints as seen by Envoy. +*/}} +{{- define "supersonic.healthyReplicasExpr" -}} +max(envoy_cluster_membership_healthy{release=~"{{ include "supersonic.name" . }}", envoy_cluster_name="triton_grpc_service"}) +{{- end -}} + +{{/* +Get default scaling metric (extensive form: replicas needed) */}} {{- define "supersonic.defaultMetric" -}} {{- if not ( eq .Values.serverLoadMetric "" ) }} {{- printf "%s" .Values.serverLoadMetric -}} {{- else }} -sum by (release) ( - rate(nv_inference_queue_duration_us{release=~"{{ include "supersonic.name" . }}"}[30s]) -) - / -sum by (release) ( - (rate(nv_inference_exec_count{release=~"{{ include "supersonic.name" . }}"}[30s]) * 1000) + 0.001 +{{- $w := include "supersonic.rateInterval" . }} +sum(rate(envoy_cluster_upstream_rq_time_sum{release=~"{{ include "supersonic.name" . }}", envoy_cluster_name="triton_grpc_service"}[{{ $w }}])) / 1e3 +* scalar(clamp_min({{ include "supersonic.healthyReplicasExpr" . }}, 1)) +/ clamp_min( + clamp_min( + ( + sum(rate(nv_inference_request_duration_us{release=~"{{ include "supersonic.name" . }}"}[{{ $w }}])) + - + sum(rate(nv_inference_queue_duration_us{release=~"{{ include "supersonic.name" . }}"}[{{ $w }}])) + ) / 1e6, + scalar({{ include "supersonic.healthyReplicasExpr" . }}) + ), + 1 + ) +{{- end }} +{{- end }} + +{{/* +Get admission metric (intensive form: load per healthy replica) +*/}} +{{- define "supersonic.admissionMetric" -}} +{{- if not ( eq .Values.serverLoadMetric "" ) }} + {{- printf "%s" .Values.serverLoadMetric -}} +{{- else }} +( +{{- include "supersonic.defaultMetric" . }} ) +/ scalar(clamp_min({{ include "supersonic.healthyReplicasExpr" . }}, 1)) {{- end }} {{- end }} {{/* -Get server load threshold (defaults to 100 if not set) +Get scaling threshold (defaults to 2 if not set) */}} {{- define "supersonic.defaultThreshold" -}} -{{- default 100 .Values.serverLoadThreshold -}} -{{- end -}} \ No newline at end of file +{{- default 2 .Values.serverLoadThreshold -}} +{{- end -}} + +{{/* +Get admission threshold for the Envoy rate limiter (defaults to 3 if not set) +*/}} +{{- define "supersonic.admissionThreshold" -}} +{{- default 3 .Values.serverAdmissionThreshold -}} +{{- end -}} diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index fa74e4fb..6dc9ef5f 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -317,8 +317,8 @@ data: envoy-filter.lua: |- {{- /* Read and process the Lua configuration file */}} {{- $luaConfig := $.Files.Get .Values.envoy.rate_limiter.prometheus_based.luaConfig | nindent 4 }} - {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_METRIC" (include "supersonic.defaultMetric" . | quote) }} - {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_THRESHOLD" (quote .Values.serverLoadThreshold) }} + {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_METRIC" (include "supersonic.admissionMetric" . | quote) }} + {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_THRESHOLD" (include "supersonic.admissionThreshold" . | quote) }} {{- $luaConfig = $luaConfig | replace "PROMETHEUS_SCHEME" (include "supersonic.prometheusScheme" .) }} {{- $luaConfig = $luaConfig | replace "PROMETHEUS_HOST" (include "supersonic.prometheusHost" .) }} {{- $luaConfig = $luaConfig | replace "PROMETHEUS_PORT" (include "supersonic.prometheusPort" .) }} diff --git a/helm/supersonic/templates/keda/so.yaml b/helm/supersonic/templates/keda/so.yaml index 9d412962..31c137d7 100644 --- a/helm/supersonic/templates/keda/so.yaml +++ b/helm/supersonic/templates/keda/so.yaml @@ -47,11 +47,11 @@ spec: triggers: - type: prometheus - metricType: Value + metricType: {{ .Values.keda.metricType | default "AverageValue" }} metadata: serverAddress: {{ include "supersonic.prometheusUrl" . }} metricName: autoscaler-metric - threshold: {{ .Values.serverLoadThreshold | quote }} + threshold: {{ include "supersonic.defaultThreshold" . | quote }} ignoreNullValues: "true" query: |- {{ include "supersonic.defaultMetric" . | nindent 8 }} diff --git a/helm/supersonic/values.schema.json b/helm/supersonic/values.schema.json index 3d64c1d7..dda8f5b0 100644 --- a/helm/supersonic/values.schema.json +++ b/helm/supersonic/values.schema.json @@ -11,6 +11,12 @@ "serverLoadThreshold": { "type": "integer" }, + "serverLoadRateInterval": { + "type": "string" + }, + "serverAdmissionThreshold": { + "type": "integer" + }, "scaleFromZero": { "type": "object", "properties": { @@ -512,6 +518,9 @@ "enabled": { "type": "boolean" }, + "metricType": { + "type": "string" + }, "minReplicaCount": { "type": "integer" }, @@ -570,6 +579,7 @@ "cooldownPeriod", "enabled", "maxReplicaCount", + "metricType", "minReplicaCount", "pollingInterval", "scaleDown", @@ -1350,7 +1360,9 @@ "nameOverride", "prometheus", "scaleFromZero", + "serverAdmissionThreshold", "serverLoadMetric", + "serverLoadRateInterval", "serverLoadThreshold", "triton" ] diff --git a/helm/supersonic/values.yaml b/helm/supersonic/values.yaml index 037b89ac..508745ea 100644 --- a/helm/supersonic/values.yaml +++ b/helm/supersonic/values.yaml @@ -3,12 +3,26 @@ # -- Unique identifier of SuperSONIC instance (equal to release name by default) nameOverride: "" -# -- A metric used by both KEDA autoscaler and Envoy's prometheus-based rate limiter. -## Default metric (inference queue latency) is defined in templates/_helpers.tpl +# -- Prometheus query used by the KEDA autoscaler and (per healthy replica) by Envoy's +# prometheus-based rate limiter. Leave empty for the default metric: requests in flight +# between Envoy and Triton divided by requests being executed, i.e. how many replicas +# the current load needs. Reads ~1 with no queueing, grows linearly with overload, and +# has no model-specific constants. See templates/_helpers/_scaling-metric.tpl and the +# configuration guide. A custom query set here is used verbatim by both consumers. serverLoadMetric: "" -# -- Threshold for the metric -serverLoadThreshold: 100 +# -- Threshold for the scaling metric. KEDA scales to ceil(metric / threshold): +# 2 targets "waiting = serving" per replica; 1.5 trades GPUs for latency. +serverLoadThreshold: 2 + +# -- Range-vector window used by rate() in the default metric. +# Keep it at or above 4x your Prometheus scrape interval. +serverLoadRateInterval: "1m" + +# -- Threshold for Envoy's prometheus-based rate limiter (scaling metric per healthy +# replica). Kept above serverLoadThreshold so new clients are rejected only when +# scaling can no longer keep up. +serverAdmissionThreshold: 3 # -- On RepositoryIndex, scale Triton to at least max(1, keda.minReplicaCount) replicas # and return the index only after Envoy has a healthy Triton upstream. @@ -226,6 +240,11 @@ keda: # new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. enabled: false + # -- How the HPA interprets the scaling metric. AverageValue (the default) treats it + # as fleet-wide "replicas needed": desired = ceil(metric / threshold). Set to Value + # for a custom per-replica metric: desired = ceil(current_replicas * metric / threshold). + metricType: AverageValue + # -- Minimum and maximum number of Triton servers. # Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled). # With scaleFromZero, a RepositoryIndex request scales Triton to max(1, minReplicaCount), diff --git a/values/values-geddes-cms.yaml b/values/values-geddes-cms.yaml index f113b11e..34258897 100644 --- a/values/values-geddes-cms.yaml +++ b/values/values-geddes-cms.yaml @@ -5,8 +5,6 @@ # Based on Purdue AF production values: # https://github.com/PurdueAF/purdue-af/blob/main/apps/sonic/supersonic/values.yaml -serverLoadThreshold: 100 - scaleFromZero: enabled: true readyTimeoutSeconds: 300 diff --git a/values/values-nautilus-atlas.yaml b/values/values-nautilus-atlas.yaml index 146963eb..709dbdec 100644 --- a/values/values-nautilus-atlas.yaml +++ b/values/values-nautilus-atlas.yaml @@ -1,5 +1,3 @@ -serverLoadThreshold: 100 - triton: name: triton-atlas image: milescb/traccc-aas:v1.1 diff --git a/values/values-nautilus-cms.yaml b/values/values-nautilus-cms.yaml index db519037..bcbcc669 100644 --- a/values/values-nautilus-cms.yaml +++ b/values/values-nautilus-cms.yaml @@ -1,5 +1,3 @@ -serverLoadThreshold: 100 - triton: replicas: 5 # image: fastml/triton-torchgeo:21.02-py3-geometric # run2 diff --git a/values/values-nautilus-icecube.yaml b/values/values-nautilus-icecube.yaml index 2686f64b..d17449d3 100644 --- a/values/values-nautilus-icecube.yaml +++ b/values/values-nautilus-icecube.yaml @@ -1,5 +1,3 @@ -serverLoadThreshold: 100 - triton: image: nvcr.io/nvidia/tritonserver:26.08-py3 affinity: