fastmachinelearning · kondratyevd · Sep 8, 2026 · Sep 8, 2026 · Sep 7, 2026 · Sep 8, 2026
diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml
@@ -34,7 +34,7 @@ jobs:
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo update
           kubectl create namespace monitoring
-          helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+          helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
 
       - name: Install KEDA Autoscaler
         run: |

diff --git a/.github/workflows/ci-full.yaml b/.github/workflows/ci-full.yaml
@@ -34,7 +34,7 @@ jobs:
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo update
           kubectl create namespace monitoring
-          helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+          helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
 
       - name: Install KEDA Autoscaler
         run: |

diff --git a/.github/workflows/ci-local.sh b/.github/workflows/ci-local.sh
@@ -53,6 +53,7 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c
 helm repo update
 kubectl create namespace monitoring
 helm install prometheus-operator prometheus-community/kube-prometheus-stack \
+  --version 89.2.4 \
   --namespace monitoring \
   --set prometheusOperator.createCustomResource=false \
   --set defaultRules.create=false \

diff --git a/README.md b/README.md
@@ -55,7 +55,7 @@ Currently, SuperSONIC supports the following functionality:
     helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
     helm repo update
     kubectl create namespace monitoring
-    helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+    helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
     ```
     </details>
   - [KEDA](https://keda.sh) CRDs (only if using autoscaling)

diff --git a/docs/.values-table.md b/docs/.values-table.md
@@ -3,8 +3,10 @@
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | nameOverride | string | `""` | Unique identifier of SuperSONIC instance (equal to release name by default) |
-| serverLoadMetric | string | `""` | A metric used by both KEDA autoscaler and Envoy's prometheus-based rate limiter. # Default metric (inference queue latency) is defined in templates/_helpers.tpl |
-| serverLoadThreshold | int | `100` | Threshold for the metric |
+| serverLoadMetric | string | `""` | Prometheus query used by the KEDA autoscaler and (per healthy replica) by Envoy's prometheus-based rate limiter. Leave empty for the default metric: requests in flight between Envoy and Triton divided by requests being executed, i.e. how many replicas the current load needs. Reads ~1 with no queueing, grows linearly with overload, and has no model-specific constants. See templates/_helpers/_scaling-metric.tpl and the configuration guide. A custom query set here is used verbatim by both consumers. |
+| serverLoadThreshold | int | `2` | Threshold for the scaling metric. KEDA scales to ceil(metric / threshold): 2 targets "waiting = serving" per replica; 1.5 trades GPUs for latency. |
+| serverLoadRateInterval | string | `"1m"` | Range-vector window used by rate() in the default metric. Keep it at or above 4x your Prometheus scrape interval. |
+| serverAdmissionThreshold | int | `3` | Threshold for Envoy's prometheus-based rate limiter (scaling metric per healthy replica). Kept above serverLoadThreshold so new clients are rejected only when scaling can no longer keep up. |
 | scaleFromZero | object | `{"admissionImage":"python:3.14-slim","enabled":false,"holdMinReplicasSeconds":300,"readyTimeoutSeconds":300}` | On RepositoryIndex, scale Triton to at least max(1, keda.minReplicaCount) replicas and return the index only after Envoy has a healthy Triton upstream. Requires keda.enabled and envoy.enabled. |
 | scaleFromZero.enabled | bool | `false` | Enable scale from zero |
 | scaleFromZero.readyTimeoutSeconds | int | `300` | Seconds to wait for a healthy Triton upstream before rejecting RepositoryIndex |
@@ -60,6 +62,7 @@
 | envoy.auth.url | string | `""` |  |
 | envoy.auth.port | int | `443` |  |
 | keda.enabled | bool | `false` | Enable autoscaling (requires Prometheus to also be enabled). Autoscaling will be based on the metric from parameter ``serverLoadMetric``; new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. |
+| keda.metricType | string | `"AverageValue"` | How the HPA interprets the scaling metric. AverageValue (the default) treats it as fleet-wide "replicas needed": desired = ceil(metric / threshold). Set to Value for a custom per-replica metric: desired = ceil(current_replicas * metric / threshold). |
 | keda.minReplicaCount | int | `1` | Minimum and maximum number of Triton servers. Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled). With scaleFromZero, a RepositoryIndex request scales Triton to max(1, minReplicaCount), and upgrades keep the live ScaledObject minReplicaCount. |
 | keda.maxReplicaCount | int | `2` |  |
 | keda.pollingInterval | int | `30` | How often KEDA polls Prometheus |

diff --git a/docs/configuration-guide.rst b/docs/configuration-guide.rst
@@ -282,20 +282,60 @@ Prometheus is needed to scrape metrics for monitoring, as well as for the rate l
 8. (Optional) Configure Metrics for Scaling and Rate Limiting
 ===============================================================
 
-Both the rate limiter and the autoscaler are currently configured to use the same Prometheus metric and threshold.
-They are defined in the ``serverLoadMetric`` and ``serverLoadThreshold`` parameters at the root level of the values file.
-The default metric is the inference queue time at the Triton servers, as defined in
-`here <https://github.com/fastmachinelearning/SuperSONIC/blob/main/helm/supersonic/templates/_scaling-metric.tpl>`_.
-
-When the metric value exceeds the threshold, the following happens:
-
-- Autoscaler scales up the number of Triton servers if possible.
-- Envoy proxy rejects new ``RepositoryIndex`` requests.
-
-The pre-configured Grafana dashboard contains a graph of this metric, entitled "Server Load Metric".
-The Prometheus query for the graph is automatically inferred from the value of ``serverLoadMetric`` parameter.
-The graph also displays the threshold value defined in ``serverLoadThreshold`` parameter.
-
+The autoscaler and the Prometheus-based rate limiter are driven by one Prometheus
+query, set by ``serverLoadMetric`` at the root of the values file and rendered in
+``templates/_helpers/_scaling-metric.tpl``.
+
+The default metric
+-------------------
+
+By default, SuperSONIC estimates **how many Triton replicas the current in-flight
+work needs**::
+
+   R_needed = L_envoy / max(L_service / R_healthy, 1)
+
+Each input is measured — the ``rate()`` of a cumulative time counter equals the
+mean number of requests inside that stage:
+
+- ``L_envoy`` — requests in flight between Envoy and Triton (queued, executing,
+  or on the wire): ``sum(rate(envoy_cluster_upstream_rq_time_sum{...}[1m])) / 1e3``.
+- ``L_service`` — requests being executed across all models and pods:
+  ``sum(rate(nv_inference_request_duration_us − nv_inference_queue_duration_us)) / 1e6``.
+  Models are weighted by the time they consume, so the metric has no
+  model-specific constants and one threshold works for any mixture.
+- ``R_healthy`` — Triton endpoints Envoy routes to:
+  ``max(envoy_cluster_membership_healthy{...})``.
+
+In PromQL, ``clamp_min(v, s)`` spells ``max(v, s)``. The floors encode that a
+healthy replica can always execute at least one request (which lets an
+underutilized fleet scale down), and that at zero replicas the metric reads
+"requests in flight" instead of dividing by zero.
+
+Per healthy replica, the metric is the inflation clients experience: 1 means
+every in-flight request is being executed, 2 means requests wait as long as they
+are served, and it keeps growing linearly with overload.
+
+Thresholds
+-----------
+
+- ``serverLoadThreshold`` (default ``2``) — KEDA scales to
+  ``ceil(metric / threshold)`` (``metricType: AverageValue``). ``2`` targets
+  "waiting ≈ serving"; ``1.5`` trades GPUs for latency. The unloaded floor is
+  ~1.2–1.3 (network transit), so values below that over-provision.
+- ``serverAdmissionThreshold`` (default ``3``) — Envoy rejects new
+  ``RepositoryIndex`` requests when the per-replica metric exceeds it. Kept
+  above ``serverLoadThreshold``: the autoscaler settles near its threshold, and
+  gating admission there would reject clients during normal operation.
+- ``serverLoadRateInterval`` (default ``1m``) — the ``rate()`` window; keep it
+  at or above 4x the Prometheus scrape interval.
+
+Custom metrics
+---------------
+
+If ``serverLoadMetric`` is set, it is used **verbatim** by both consumers: KEDA
+compares it against ``serverLoadThreshold`` and the rate limiter against
+``serverAdmissionThreshold``. Set ``keda.metricType: Value`` for per-replica
+custom metrics.
 
 9. (Optional) Deploy Grafana Dashboard
 ==========================================

diff --git a/helm/supersonic/README.md b/helm/supersonic/README.md
@@ -55,7 +55,7 @@ Currently, SuperSONIC supports the following functionality:
     helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
     helm repo update
     kubectl create namespace monitoring
-    helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+    helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
     ```
     </details>
   - [KEDA](https://keda.sh) CRDs (only if using autoscaling)

diff --git a/helm/supersonic/templates/NOTES.txt b/helm/supersonic/templates/NOTES.txt
@@ -24,7 +24,9 @@ SuperSONIC chart successfully installed!
 
 Scaling metric:{{ if not ( eq .Values.serverLoadMetric "" ) }}    {{ .Values.serverLoadMetric }}{{ else }}{{ include "supersonic.defaultMetric" . | nindent 4 }}{{ end }}
 
-Scaling threshold: {{ include "supersonic.defaultThreshold" . }}
+Scaling threshold (KEDA): {{ include "supersonic.defaultThreshold" . }}
+
+Admission threshold (Envoy rate limiter): {{ include "supersonic.admissionThreshold" . }}
 
 ┌-----------------------------------------------------------------------------┐
 | Documentation:           https://fastmachinelearning.org/SuperSONIC

diff --git a/helm/supersonic/templates/_helpers/_scaling-metric.tpl b/helm/supersonic/templates/_helpers/_scaling-metric.tpl
@@ -1,23 +1,93 @@
 {{/*
-Get default scaling metric
+Scaling and admission metrics.
+
+The default scaling metric estimates how many replicas the current in-flight
+work needs:
+
+  R_needed = L_envoy / max(L_service / R_healthy, 1)
+
+  L_envoy   - mean requests in flight between Envoy and Triton: rate of
+              Envoy's cumulative upstream request-time counter (ms -> /1e3).
+  L_service - mean requests being executed across all models and pods: rate
+              of Triton's request-duration minus queue-duration counters
+              (us -> /1e6).
+  R_healthy - Triton endpoints Envoy routes to (max across Envoy pods).
+
+clamp_min(v, s) is PromQL for max(v, s). The floors encode that a healthy
+replica can always execute at least one request, and that at zero replicas
+the metric reads "requests in flight" instead of dividing by zero.
+
+KEDA consumes "supersonic.defaultMetric" (the form above) with metricType
+AverageValue: desired = ceil(R_needed / serverLoadThreshold). The Envoy Lua
+filter consumes "supersonic.admissionMetric" (the same per healthy replica)
+and rejects RepositoryIndex above "supersonic.admissionThreshold".
+
+If .Values.serverLoadMetric is set, both helpers return it verbatim.
+*/}}
+
+{{/*
+Range-vector window for rate() in the default metric.
+Keep it at >= 4x the Prometheus scrape interval.
+*/}}
+{{- define "supersonic.rateInterval" -}}
+{{- default "1m" .Values.serverLoadRateInterval -}}
+{{- end -}}
+
+{{/*
+Healthy Triton endpoints as seen by Envoy.
+*/}}
+{{- define "supersonic.healthyReplicasExpr" -}}
+max(envoy_cluster_membership_healthy{release=~"{{ include "supersonic.name" . }}", envoy_cluster_name="triton_grpc_service"})
+{{- end -}}
+
+{{/*
+Get default scaling metric (extensive form: replicas needed)
 */}}
 {{- define "supersonic.defaultMetric" -}}
 {{- if not ( eq .Values.serverLoadMetric "" ) }}
   {{- printf "%s" .Values.serverLoadMetric -}}
 {{- else }}
-sum by (release) (
-    rate(nv_inference_queue_duration_us{release=~"{{ include "supersonic.name" . }}"}[30s])
-)
-  /
-sum by (release) (
-    (rate(nv_inference_exec_count{release=~"{{ include "supersonic.name" . }}"}[30s]) * 1000) + 0.001
+{{- $w := include "supersonic.rateInterval" . }}
+sum(rate(envoy_cluster_upstream_rq_time_sum{release=~"{{ include "supersonic.name" . }}", envoy_cluster_name="triton_grpc_service"}[{{ $w }}])) / 1e3
+* scalar(clamp_min({{ include "supersonic.healthyReplicasExpr" . }}, 1))
+/ clamp_min(
+    clamp_min(
+      (
+        sum(rate(nv_inference_request_duration_us{release=~"{{ include "supersonic.name" . }}"}[{{ $w }}]))
+        -
+        sum(rate(nv_inference_queue_duration_us{release=~"{{ include "supersonic.name" . }}"}[{{ $w }}]))
+      ) / 1e6,
+      scalar({{ include "supersonic.healthyReplicasExpr" . }})
+    ),
+    1
+  )
+{{- end }}
+{{- end }}
+
+{{/*
+Get admission metric (intensive form: load per healthy replica)
+*/}}
+{{- define "supersonic.admissionMetric" -}}
+{{- if not ( eq .Values.serverLoadMetric "" ) }}
+  {{- printf "%s" .Values.serverLoadMetric -}}
+{{- else }}
+(
+{{- include "supersonic.defaultMetric" . }}
 )
+/ scalar(clamp_min({{ include "supersonic.healthyReplicasExpr" . }}, 1))
 {{- end }}
 {{- end }}
 
 {{/*
-Get server load threshold (defaults to 100 if not set)
+Get scaling threshold (defaults to 2 if not set)
 */}}
 {{- define "supersonic.defaultThreshold" -}}
-{{- default 100 .Values.serverLoadThreshold -}}
-{{- end -}} 
+{{- default 2 .Values.serverLoadThreshold -}}
+{{- end -}}
+
+{{/*
+Get admission threshold for the Envoy rate limiter (defaults to 3 if not set)
+*/}}
+{{- define "supersonic.admissionThreshold" -}}
+{{- default 3 .Values.serverAdmissionThreshold -}}
+{{- end -}}
diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml
@@ -317,8 +317,8 @@ data:
   envoy-filter.lua: |-
     {{- /* Read and process the Lua configuration file */}}
     {{- $luaConfig := $.Files.Get .Values.envoy.rate_limiter.prometheus_based.luaConfig | nindent 4 }}
-    {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_METRIC" (include "supersonic.defaultMetric" . | quote) }}
-    {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_THRESHOLD" (quote .Values.serverLoadThreshold) }}
+    {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_METRIC" (include "supersonic.admissionMetric" . | quote) }}
+    {{- $luaConfig = $luaConfig | replace "SERVER_LOAD_THRESHOLD" (include "supersonic.admissionThreshold" . | quote) }}
     {{- $luaConfig = $luaConfig | replace "PROMETHEUS_SCHEME" (include "supersonic.prometheusScheme" .) }}
     {{- $luaConfig = $luaConfig | replace "PROMETHEUS_HOST" (include "supersonic.prometheusHost" .) }}
     {{- $luaConfig = $luaConfig | replace "PROMETHEUS_PORT" (include "supersonic.prometheusPort" .) }}

diff --git a/helm/supersonic/templates/keda/so.yaml b/helm/supersonic/templates/keda/so.yaml
@@ -47,11 +47,11 @@ spec:
 
   triggers:
   - type: prometheus
-    metricType: Value
+    metricType: {{ .Values.keda.metricType | default "AverageValue" }}
     metadata:
       serverAddress: {{ include "supersonic.prometheusUrl" . }}
       metricName: autoscaler-metric
-      threshold: {{ .Values.serverLoadThreshold | quote }}
+      threshold: {{ include "supersonic.defaultThreshold" . | quote }}
       ignoreNullValues: "true"
       query: |-
         {{ include "supersonic.defaultMetric" . | nindent 8 }}

diff --git a/helm/supersonic/values.schema.json b/helm/supersonic/values.schema.json
@@ -11,6 +11,12 @@
     "serverLoadThreshold": {
       "type": "integer"
     },
+    "serverLoadRateInterval": {
+      "type": "string"
+    },
+    "serverAdmissionThreshold": {
+      "type": "integer"
+    },
     "scaleFromZero": {
       "type": "object",
       "properties": {
@@ -512,6 +518,9 @@
         "enabled": {
           "type": "boolean"
         },
+        "metricType": {
+          "type": "string"
+        },
         "minReplicaCount": {
           "type": "integer"
         },
@@ -570,6 +579,7 @@
         "cooldownPeriod",
         "enabled",
         "maxReplicaCount",
+        "metricType",
         "minReplicaCount",
         "pollingInterval",
         "scaleDown",
@@ -1350,7 +1360,9 @@
     "nameOverride",
     "prometheus",
     "scaleFromZero",
+    "serverAdmissionThreshold",
     "serverLoadMetric",
+    "serverLoadRateInterval",
     "serverLoadThreshold",
     "triton"
   ]

diff --git a/helm/supersonic/values.yaml b/helm/supersonic/values.yaml
@@ -3,12 +3,26 @@
 # -- Unique identifier of SuperSONIC instance (equal to release name by default)
 nameOverride: ""
 
-# -- A metric used by both KEDA autoscaler and Envoy's prometheus-based rate limiter.
-## Default metric (inference queue latency) is defined in templates/_helpers.tpl
+# -- Prometheus query used by the KEDA autoscaler and (per healthy replica) by Envoy's
+# prometheus-based rate limiter. Leave empty for the default metric: requests in flight
+# between Envoy and Triton divided by requests being executed, i.e. how many replicas
+# the current load needs. Reads ~1 with no queueing, grows linearly with overload, and
+# has no model-specific constants. See templates/_helpers/_scaling-metric.tpl and the
+# configuration guide. A custom query set here is used verbatim by both consumers.
 serverLoadMetric: ""
 
-# -- Threshold for the metric
-serverLoadThreshold: 100
+# -- Threshold for the scaling metric. KEDA scales to ceil(metric / threshold):
+# 2 targets "waiting = serving" per replica; 1.5 trades GPUs for latency.
+serverLoadThreshold: 2
+
+# -- Range-vector window used by rate() in the default metric.
+# Keep it at or above 4x your Prometheus scrape interval.
+serverLoadRateInterval: "1m"
+
+# -- Threshold for Envoy's prometheus-based rate limiter (scaling metric per healthy
+# replica). Kept above serverLoadThreshold so new clients are rejected only when
+# scaling can no longer keep up.
+serverAdmissionThreshold: 3
 
 # -- On RepositoryIndex, scale Triton to at least max(1, keda.minReplicaCount) replicas
 # and return the index only after Envoy has a healthy Triton upstream.
@@ -226,6 +240,11 @@ keda:
   # new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``.
   enabled: false
 
+  # -- How the HPA interprets the scaling metric. AverageValue (the default) treats it
+  # as fleet-wide "replicas needed": desired = ceil(metric / threshold). Set to Value
+  # for a custom per-replica metric: desired = ceil(current_replicas * metric / threshold).
+  metricType: AverageValue
+
   # -- Minimum and maximum number of Triton servers.
   # Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled).
   # With scaleFromZero, a RepositoryIndex request scales Triton to max(1, minReplicaCount),

diff --git a/values/values-geddes-cms.yaml b/values/values-geddes-cms.yaml
@@ -5,8 +5,6 @@
 # Based on Purdue AF production values:
 # https://github.com/PurdueAF/purdue-af/blob/main/apps/sonic/supersonic/values.yaml
 
-serverLoadThreshold: 100
-
 scaleFromZero:
   enabled: true
   readyTimeoutSeconds: 300