Support inference servers other than Triton, starting with Nereid

.github/workflows/ci-external-config.yaml

@@ -31,10 +31,13 @@ jobs:
 
       - name: Install Prometheus Operator CRDs
         run: |
+          # Pinned: 90.0.0 refuses to template the control-plane ServiceMonitors
+          # unless prometheus.enabled is true, and this step wants only the
+          # operator and its CRDs. Bump deliberately, not implicitly.
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo update
           kubectl create namespace monitoring
-          helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+          helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
 
       - name: Install KEDA Autoscaler
         run: |
@@ -68,10 +71,10 @@ jobs:
         run: |
           kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 120s -n cms
 
-      - name: Triton server ready
+      - name: Inference server ready
         run: |
-          kubectl describe pod -l app.kubernetes.io/component=triton -n cms
-          kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=triton --timeout 500s -n cms
+          kubectl describe pod -l app.kubernetes.io/component=inference-server -n cms
+          kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=inference-server --timeout 500s -n cms
 
       - name: Validate Deployment
         run: |

.github/workflows/ci-full.yaml

@@ -31,10 +31,13 @@ jobs:
 
       - name: Install Prometheus Operator CRDs
         run: |
+          # Pinned: 90.0.0 refuses to template the control-plane ServiceMonitors
+          # unless prometheus.enabled is true, and this step wants only the
+          # operator and its CRDs. Bump deliberately, not implicitly.
           helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
           helm repo update
           kubectl create namespace monitoring
-          helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+          helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
 
       - name: Install KEDA Autoscaler
         run: |
@@ -82,19 +85,19 @@ jobs:
           kubectl wait --for condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 180s -n cms
           kubectl wait --for condition=Ready so -l app.kubernetes.io/component=keda --timeout 180s -n cms
 
-      - name: Triton idle at zero replicas
+      - name: Inference server idle at zero replicas
         run: |
-          echo "Waiting for Triton Deployment spec.replicas=0..."
+          echo "Waiting for inference server Deployment spec.replicas=0..."
           for i in $(seq 1 36); do
-            replicas=$(kubectl get deploy -l app.kubernetes.io/component=triton -n cms -o jsonpath='{.items[0].spec.replicas}')
-            echo "Triton spec.replicas=${replicas:-unset}"
+            replicas=$(kubectl get deploy -l app.kubernetes.io/component=inference-server -n cms -o jsonpath='{.items[0].spec.replicas}')
+            echo "Inference server spec.replicas=${replicas:-unset}"
             if [ "${replicas}" = "0" ]; then
-              kubectl get deploy,pod -l app.kubernetes.io/component=triton -n cms
+              kubectl get deploy,pod -l app.kubernetes.io/component=inference-server -n cms
               exit 0
             fi
             sleep 5
           done
-          echo "Triton did not scale to 0 replicas"
+          echo "Inference server did not scale to 0 replicas"
           exit 1
 
       - name: Validate Deployment

.github/workflows/ci-local.sh

@@ -53,6 +53,7 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c
 helm repo update
 kubectl create namespace monitoring
 helm install prometheus-operator prometheus-community/kube-prometheus-stack \
+  --version 89.2.4 \
   --namespace monitoring \
   --set prometheusOperator.createCustomResource=false \
   --set defaultRules.create=false \
@@ -100,15 +101,15 @@ echo "Waiting for KEDA Autoscaler to be ready..."
 kubectl wait --for=condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 120s -n cms
 kubectl wait --for=condition=Ready so -l app.kubernetes.io/component=keda --timeout 120s -n cms
 
-echo "Waiting for Triton Deployment spec.replicas=0..."
+echo "Waiting for inference server Deployment spec.replicas=0..."
 for i in $(seq 1 36); do
-  replicas=$(kubectl get deploy -l app.kubernetes.io/component=triton -n cms -o jsonpath='{.items[0].spec.replicas}')
-  echo "Triton spec.replicas=${replicas:-unset}"
+  replicas=$(kubectl get deploy -l app.kubernetes.io/component=inference-server -n cms -o jsonpath='{.items[0].spec.replicas}')
+  echo "Inference server spec.replicas=${replicas:-unset}"
   if [ "${replicas}" = "0" ]; then
     break
   fi
   if [ "$i" -eq 36 ]; then
-    echo "Triton did not scale to 0 replicas"
+    echo "Inference server did not scale to 0 replicas"
     bash .github/scripts/k8s-diagnostics.sh cms
     exit 1
   fi

.github/workflows/ci-nereid.yaml

@@ -0,0 +1,98 @@
+name: ci [nereid]
+
+on:
+  push:
+    branches:
+      - "**"
+  pull_request:
+    branches:
+      - "main"
+
+jobs:
+  deploy-nereid:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # TODO: drop this gate once nereid-server publishes a release. The
+      # repository has no tags and its GHCR package is unpublished, so the
+      # image reference cannot resolve yet. Skipping loudly beats a job that
+      # is red for a reason unrelated to this chart.
+      - name: Check whether the Nereid image is published
+        id: image
+        run: |
+          IMAGE=$(grep -m1 'image: ghcr.io/ngpaladi/nereid-server' tests/values-nereid-ci.yaml | awk '{print $2}')
+          echo "ref=$IMAGE" >> "$GITHUB_OUTPUT"
+          if docker manifest inspect "$IMAGE" > /dev/null 2>&1; then
+            echo "available=true" >> "$GITHUB_OUTPUT"
+            echo "Nereid image $IMAGE is available."
+          else
+            echo "available=false" >> "$GITHUB_OUTPUT"
+            echo "::notice::Nereid image $IMAGE is not published yet; skipping the deployment test."
+          fi
+
+      - name: Set up Kubernetes cluster with Kind
+        if: steps.image.outputs.available == 'true'
+        uses: helm/kind-action@v1.6.0
+        with:
+          cluster_name: gh-k8s-cluster
+
+      - name: Set up Helm
+        if: steps.image.outputs.available == 'true'
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.12.0
+
+      - name: Create CMS namespace
+        if: steps.image.outputs.available == 'true'
+        run: kubectl create namespace cms
+
+      - name: Create model fixture
+        if: steps.image.outputs.available == 'true'
+        run: kubectl apply -f tests/nereid-model-fixture.yaml
+
+      - name: Deploy Helm chart with Nereid
+        if: steps.image.outputs.available == 'true'
+        run: |
+          # Every repository Chart.yaml lists a dependency from, even though
+          # this job disables those subcharts: `helm dependency build` still
+          # has to resolve all of them.
+          helm repo add grafana https://grafana.github.io/helm-charts
+          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+          helm repo update
+          helm dependency build ./helm/supersonic
+          helm upgrade --install supersonic ./helm/supersonic \
+            --values tests/values-nereid-ci.yaml -n cms
+
+      - name: Nereid server ready
+        if: steps.image.outputs.available == 'true'
+        run: |
+          kubectl wait --for condition=Ready pod \
+            -l app.kubernetes.io/component=inference-server --timeout 300s -n cms
+
+      - name: Envoy proxy ready
+        if: steps.image.outputs.available == 'true'
+        run: |
+          kubectl wait --for condition=Ready pod \
+            -l app.kubernetes.io/component=envoy --timeout 180s -n cms
+
+      - name: Validate deployment
+        if: steps.image.outputs.available == 'true'
+        run: kubectl get all -n cms
+
+      - name: Run inference through Envoy
+        if: steps.image.outputs.available == 'true'
+        run: |
+          kubectl apply -f tests/nereid-infer-job.yaml
+          bash .github/scripts/wait-for-job.sh nereid-infer-job cms 600
+
+      - name: Failure diagnostics
+        if: failure() && steps.image.outputs.available == 'true'
+        uses: ./.github/actions/k8s-diagnostics
+        with:
+          namespace: cms
+
+      - name: Cleanup
+        if: always() && steps.image.outputs.available == 'true'
+        run: kind delete cluster --name gh-k8s-cluster

.github/workflows/yaml-to-schema.py

@@ -4,6 +4,25 @@
 import json
 from genson import SchemaBuilder
 
+
+def drop_required(node):
+    """Remove genson's inferred "required" lists.
+
+    genson marks every key it saw as required, and it only ever sees
+    values.yaml -- so "required" is just a copy of the defaults. Helm merges
+    those defaults into every release, which means the constraint cannot catch
+    a missing key; it only fires when a values file deliberately clears one
+    (`command: null` to swap a probe handler, say) and rejects it.
+    """
+    if isinstance(node, dict):
+        node.pop("required", None)
+        for value in node.values():
+            drop_required(value)
+    elif isinstance(node, list):
+        for value in node:
+            drop_required(value)
+
+
 def main():
     input_file = sys.argv[1]
     output_file = sys.argv[2]
@@ -14,6 +33,7 @@ def main():
     builder = SchemaBuilder()
     builder.add_object(data)
     schema = builder.to_schema()
+    drop_required(schema)
 
     with open(output_file, 'w') as f:
         json.dump(schema, f, indent=2)

CITATION.cff

@@ -32,6 +32,7 @@ abstract: >+
 keywords:
   - Kubernetes
   - NVIDIA Triton Inference Server
+  - Nereid
   - inference as a service
   - GPU
   - machine learning

README.md

@@ -18,6 +18,7 @@ applications in large high energy physics (HEP) and multi-messenger astrophysics
 
 Currently, SuperSONIC supports the following functionality:
 - GPU inference-as-a-service via [Nvidia Triton Inference Server](https://developer.nvidia.com/triton-inference-server)
+  or [Nereid](https://github.com/ngpaladi/nereid-server), selected with `inferenceServer.type`
 - Load balancing across many GPUs via [Envoy Proxy](envoyproxy.io)
 - Load-based autoscaling via [KEDA](keda.sh), including scale from zero replicas on `RepositoryIndex`
 - Monitoring via [Prometheus](https://prometheus.io) and [Grafana](https://grafana.com)
@@ -55,7 +56,7 @@ Currently, SuperSONIC supports the following functionality:
     helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
     helm repo update
     kubectl create namespace monitoring
-    helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
+    helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false
     ```
     </details>
   - [KEDA](https://keda.sh) CRDs (only if using autoscaling)
@@ -116,7 +117,7 @@ kubectl apply -f cvmfs/cvmfs-storageclass.yaml -n cvmfs-csi
 <details>
 <summary><strong>2. Install SuperSONIC with minimal configuration</strong></summary>
 
-The minimal deployment will install only a single CPU-based Triton server and an Envoy Proxy.
+The minimal deployment will install only a single CPU-based inference server (Triton by default) and an Envoy Proxy.
 We will use [`values/values-minimal.yaml`](values/values-minimal.yaml) as our minimal
 configuration file.