diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml index a0072e49..f5521b62 100644 --- a/.github/workflows/ci-external-config.yaml +++ b/.github/workflows/ci-external-config.yaml @@ -31,10 +31,13 @@ jobs: - name: Install Prometheus Operator CRDs run: | + # Pinned: 90.0.0 refuses to template the control-plane ServiceMonitors + # unless prometheus.enabled is true, and this step wants only the + # operator and its CRDs. Bump deliberately, not implicitly. helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false - name: Install KEDA Autoscaler run: | @@ -68,10 +71,10 @@ jobs: run: | kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 120s -n cms - - name: Triton server ready + - name: Inference server ready run: | - kubectl describe pod -l app.kubernetes.io/component=triton -n cms - kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=triton --timeout 500s -n cms + kubectl describe pod -l app.kubernetes.io/component=inference-server -n cms + kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=inference-server --timeout 500s -n cms - name: Validate Deployment run: | diff --git a/.github/workflows/ci-full.yaml b/.github/workflows/ci-full.yaml index 9416fc7d..bdc251c4 100644 --- a/.github/workflows/ci-full.yaml +++ b/.github/workflows/ci-full.yaml @@ -31,10 +31,13 @@ jobs: - name: Install Prometheus Operator CRDs run: | + # Pinned: 90.0.0 refuses to template the control-plane ServiceMonitors + # unless prometheus.enabled is true, and this step wants only the + # operator and its CRDs. Bump deliberately, not implicitly. helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false - name: Install KEDA Autoscaler run: | @@ -82,19 +85,19 @@ jobs: kubectl wait --for condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 180s -n cms kubectl wait --for condition=Ready so -l app.kubernetes.io/component=keda --timeout 180s -n cms - - name: Triton idle at zero replicas + - name: Inference server idle at zero replicas run: | - echo "Waiting for Triton Deployment spec.replicas=0..." + echo "Waiting for inference server Deployment spec.replicas=0..." for i in $(seq 1 36); do - replicas=$(kubectl get deploy -l app.kubernetes.io/component=triton -n cms -o jsonpath='{.items[0].spec.replicas}') - echo "Triton spec.replicas=${replicas:-unset}" + replicas=$(kubectl get deploy -l app.kubernetes.io/component=inference-server -n cms -o jsonpath='{.items[0].spec.replicas}') + echo "Inference server spec.replicas=${replicas:-unset}" if [ "${replicas}" = "0" ]; then - kubectl get deploy,pod -l app.kubernetes.io/component=triton -n cms + kubectl get deploy,pod -l app.kubernetes.io/component=inference-server -n cms exit 0 fi sleep 5 done - echo "Triton did not scale to 0 replicas" + echo "Inference server did not scale to 0 replicas" exit 1 - name: Validate Deployment diff --git a/.github/workflows/ci-local.sh b/.github/workflows/ci-local.sh index 7f255758..ded31d0a 100644 --- a/.github/workflows/ci-local.sh +++ b/.github/workflows/ci-local.sh @@ -53,6 +53,7 @@ helm repo add prometheus-community https://prometheus-community.github.io/helm-c helm repo update kubectl create namespace monitoring helm install prometheus-operator prometheus-community/kube-prometheus-stack \ + --version 89.2.4 \ --namespace monitoring \ --set prometheusOperator.createCustomResource=false \ --set defaultRules.create=false \ @@ -100,15 +101,15 @@ echo "Waiting for KEDA Autoscaler to be ready..." kubectl wait --for=condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 120s -n cms kubectl wait --for=condition=Ready so -l app.kubernetes.io/component=keda --timeout 120s -n cms -echo "Waiting for Triton Deployment spec.replicas=0..." +echo "Waiting for inference server Deployment spec.replicas=0..." for i in $(seq 1 36); do - replicas=$(kubectl get deploy -l app.kubernetes.io/component=triton -n cms -o jsonpath='{.items[0].spec.replicas}') - echo "Triton spec.replicas=${replicas:-unset}" + replicas=$(kubectl get deploy -l app.kubernetes.io/component=inference-server -n cms -o jsonpath='{.items[0].spec.replicas}') + echo "Inference server spec.replicas=${replicas:-unset}" if [ "${replicas}" = "0" ]; then break fi if [ "$i" -eq 36 ]; then - echo "Triton did not scale to 0 replicas" + echo "Inference server did not scale to 0 replicas" bash .github/scripts/k8s-diagnostics.sh cms exit 1 fi diff --git a/.github/workflows/ci-nereid.yaml b/.github/workflows/ci-nereid.yaml new file mode 100644 index 00000000..d0c2d911 --- /dev/null +++ b/.github/workflows/ci-nereid.yaml @@ -0,0 +1,98 @@ +name: ci [nereid] + +on: + push: + branches: + - "**" + pull_request: + branches: + - "main" + +jobs: + deploy-nereid: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + # TODO: drop this gate once nereid-server publishes a release. The + # repository has no tags and its GHCR package is unpublished, so the + # image reference cannot resolve yet. Skipping loudly beats a job that + # is red for a reason unrelated to this chart. + - name: Check whether the Nereid image is published + id: image + run: | + IMAGE=$(grep -m1 'image: ghcr.io/ngpaladi/nereid-server' tests/values-nereid-ci.yaml | awk '{print $2}') + echo "ref=$IMAGE" >> "$GITHUB_OUTPUT" + if docker manifest inspect "$IMAGE" > /dev/null 2>&1; then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "Nereid image $IMAGE is available." + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::notice::Nereid image $IMAGE is not published yet; skipping the deployment test." + fi + + - name: Set up Kubernetes cluster with Kind + if: steps.image.outputs.available == 'true' + uses: helm/kind-action@v1.6.0 + with: + cluster_name: gh-k8s-cluster + + - name: Set up Helm + if: steps.image.outputs.available == 'true' + uses: azure/setup-helm@v3 + with: + version: v3.12.0 + + - name: Create CMS namespace + if: steps.image.outputs.available == 'true' + run: kubectl create namespace cms + + - name: Create model fixture + if: steps.image.outputs.available == 'true' + run: kubectl apply -f tests/nereid-model-fixture.yaml + + - name: Deploy Helm chart with Nereid + if: steps.image.outputs.available == 'true' + run: | + # Every repository Chart.yaml lists a dependency from, even though + # this job disables those subcharts: `helm dependency build` still + # has to resolve all of them. + helm repo add grafana https://grafana.github.io/helm-charts + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + helm dependency build ./helm/supersonic + helm upgrade --install supersonic ./helm/supersonic \ + --values tests/values-nereid-ci.yaml -n cms + + - name: Nereid server ready + if: steps.image.outputs.available == 'true' + run: | + kubectl wait --for condition=Ready pod \ + -l app.kubernetes.io/component=inference-server --timeout 300s -n cms + + - name: Envoy proxy ready + if: steps.image.outputs.available == 'true' + run: | + kubectl wait --for condition=Ready pod \ + -l app.kubernetes.io/component=envoy --timeout 180s -n cms + + - name: Validate deployment + if: steps.image.outputs.available == 'true' + run: kubectl get all -n cms + + - name: Run inference through Envoy + if: steps.image.outputs.available == 'true' + run: | + kubectl apply -f tests/nereid-infer-job.yaml + bash .github/scripts/wait-for-job.sh nereid-infer-job cms 600 + + - name: Failure diagnostics + if: failure() && steps.image.outputs.available == 'true' + uses: ./.github/actions/k8s-diagnostics + with: + namespace: cms + + - name: Cleanup + if: always() && steps.image.outputs.available == 'true' + run: kind delete cluster --name gh-k8s-cluster diff --git a/.github/workflows/yaml-to-schema.py b/.github/workflows/yaml-to-schema.py index 6d8f6a46..7c046126 100644 --- a/.github/workflows/yaml-to-schema.py +++ b/.github/workflows/yaml-to-schema.py @@ -4,6 +4,25 @@ import json from genson import SchemaBuilder + +def drop_required(node): + """Remove genson's inferred "required" lists. + + genson marks every key it saw as required, and it only ever sees + values.yaml -- so "required" is just a copy of the defaults. Helm merges + those defaults into every release, which means the constraint cannot catch + a missing key; it only fires when a values file deliberately clears one + (`command: null` to swap a probe handler, say) and rejects it. + """ + if isinstance(node, dict): + node.pop("required", None) + for value in node.values(): + drop_required(value) + elif isinstance(node, list): + for value in node: + drop_required(value) + + def main(): input_file = sys.argv[1] output_file = sys.argv[2] @@ -14,6 +33,7 @@ def main(): builder = SchemaBuilder() builder.add_object(data) schema = builder.to_schema() + drop_required(schema) with open(output_file, 'w') as f: json.dump(schema, f, indent=2) diff --git a/CITATION.cff b/CITATION.cff index 561b4c39..88d5428b 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -32,6 +32,7 @@ abstract: >+ keywords: - Kubernetes - NVIDIA Triton Inference Server + - Nereid - inference as a service - GPU - machine learning diff --git a/README.md b/README.md index 45a9fe60..5c1d7118 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ applications in large high energy physics (HEP) and multi-messenger astrophysics Currently, SuperSONIC supports the following functionality: - GPU inference-as-a-service via [Nvidia Triton Inference Server](https://developer.nvidia.com/triton-inference-server) + or [Nereid](https://github.com/ngpaladi/nereid-server), selected with `inferenceServer.type` - Load balancing across many GPUs via [Envoy Proxy](envoyproxy.io) - Load-based autoscaling via [KEDA](keda.sh), including scale from zero replicas on `RepositoryIndex` - Monitoring via [Prometheus](https://prometheus.io) and [Grafana](https://grafana.com) @@ -55,7 +56,7 @@ Currently, SuperSONIC supports the following functionality: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false ``` - [KEDA](https://keda.sh) CRDs (only if using autoscaling) @@ -116,7 +117,7 @@ kubectl apply -f cvmfs/cvmfs-storageclass.yaml -n cvmfs-csi
2. Install SuperSONIC with minimal configuration -The minimal deployment will install only a single CPU-based Triton server and an Envoy Proxy. +The minimal deployment will install only a single CPU-based inference server (Triton by default) and an Envoy Proxy. We will use [`values/values-minimal.yaml`](values/values-minimal.yaml) as our minimal configuration file. diff --git a/docs/.values-table.md b/docs/.values-table.md index ad7c7e1e..22ba9607 100644 --- a/docs/.values-table.md +++ b/docs/.values-table.md @@ -5,29 +5,33 @@ | nameOverride | string | `""` | Unique identifier of SuperSONIC instance (equal to release name by default) | | serverLoadMetric | string | `""` | A metric used by both KEDA autoscaler and Envoy's prometheus-based rate limiter. # Default metric (inference queue latency) is defined in templates/_helpers.tpl | | serverLoadThreshold | int | `100` | Threshold for the metric | -| scaleFromZero | object | `{"admissionImage":"python:3.14-slim","enabled":false,"holdMinReplicasSeconds":300,"readyTimeoutSeconds":300}` | On RepositoryIndex, scale Triton to at least max(1, keda.minReplicaCount) replicas and return the index only after Envoy has a healthy Triton upstream. Requires keda.enabled and envoy.enabled. | +| scaleFromZero | object | `{"admissionImage":"python:3.14-slim","enabled":false,"holdMinReplicasSeconds":300,"readyTimeoutSeconds":300}` | On RepositoryIndex, scale the inference server to at least max(1, keda.minReplicaCount) replicas and return the index only after Envoy has a healthy inference server upstream. Requires keda.enabled and envoy.enabled. | | scaleFromZero.enabled | bool | `false` | Enable scale from zero | -| scaleFromZero.readyTimeoutSeconds | int | `300` | Seconds to wait for a healthy Triton upstream before rejecting RepositoryIndex | +| scaleFromZero.readyTimeoutSeconds | int | `300` | Seconds to wait for a healthy inference server upstream before rejecting RepositoryIndex | | scaleFromZero.holdMinReplicasSeconds | int | `300` | Seconds to keep KEDA minReplicaCount at the wake target after the last RepositoryIndex is answered (the hold is refreshed once the server is ready, so it is measured from readiness rather than from the wake request). Must be >= readyTimeoutSeconds (validated at render time), otherwise KEDA can scale the waking pod back to zero before RepositoryIndex is answered. | | scaleFromZero.admissionImage | string | `"python:3.14-slim"` | Image for the admission sidecar on the Envoy pod | -| triton.replicas | int | `1` | Number of Triton server instances. Unused when scaleFromZero.enabled is true. | -| triton.image | string | `"nvcr.io/nvidia/tritonserver:26.08-py3-min"` | Docker image for the Triton server | -| triton.command | list | `["/bin/sh","-c"]` | Command and arguments to run in Triton container | -| triton.args[0] | string | `"/opt/tritonserver/bin/tritonserver \\\n--model-repository=/tmp/ \\\n--log-verbose=0 \\\n--exit-timeout-secs=60\n"` | | -| triton.resources | object | `{"limits":{"cpu":1,"memory":"2G"},"requests":{"cpu":1,"memory":"2G"}}` | Resource limits and requests for each Triton instance. You can add necessary GPU request here. | -| triton.annotations | object | `{}` | Annotations for Triton pods | -| triton.nodeSelector | object | `{}` | Node selector for Triton pods | -| triton.tolerations | list | `[]` | Tolerations for Triton pods | -| triton.affinity | object | `{}` | Affinity rules for Triton pods - another way to request GPUs | -| triton.modelRepository | object | `{"enabled":false,"mountPath":""}` | Model repository configuration | -| triton.modelRepository.mountPath | string | `""` | Model repository mount path | -| triton.service.labels | object | `{}` | | -| triton.service.annotations | object | `{}` | | -| triton.service.ports | list | `[{"name":"http","port":8000,"protocol":"TCP","targetPort":8000},{"name":"grpc","port":8001,"protocol":"TCP","targetPort":8001},{"name":"metrics","port":8002,"protocol":"TCP","targetPort":8002}]` | Ports for communication with Triton servers | -| triton.readinessProbe | object | `{"command":["/bin/sh","-c","curl -sf http://localhost:8000/v2/health/ready > /dev/null && [ ! -f /tmp/shutdown ]"],"failureThreshold":10,"initialDelaySeconds":10,"periodSeconds":10,"reset":false,"successThreshold":1,"timeoutSeconds":15}` | Custom readiness probe configuration | -| triton.readinessProbe.reset | bool | `false` | If true, will reset settings to k8s defaults (other readinessProbe settings will be ignored) | -| triton.startupProbe | object | `{"failureThreshold":24,"httpGet":{"path":"/v2/health/ready","port":"http"},"initialDelaySeconds":0,"periodSeconds":10,"reset":false,"timeoutSeconds":15}` | Custom startup probe configuration | -| triton.startupProbe.reset | bool | `false` | If true, will reset settings to k8s defaults (other startupProbe settings will be ignored) | +| inferenceServer.type | string | `"triton"` | Inference server implementation. Supported values: ``triton``, ``nereid``. Nereid speaks the same KServe v2 gRPC protocol as Triton, so only the image, launch command, model configuration and probe endpoints differ between them. See ``values/values-nereid.yaml`` for a complete Nereid example. | +| inferenceServer.replicas | int | `1` | Number of inference server instances. Unused when scaleFromZero.enabled is true. | +| inferenceServer.image | string | `"nvcr.io/nvidia/tritonserver:26.08-py3-min"` | Docker image for the inference server | +| inferenceServer.command | list | `["/bin/sh","-c"]` | Command and arguments to run in the inference server container | +| inferenceServer.args[0] | string | `"/opt/tritonserver/bin/tritonserver \\\n--model-repository=/tmp/ \\\n--log-verbose=0 \\\n--exit-timeout-secs=60\n"` | | +| inferenceServer.resources | object | `{"limits":{"cpu":1,"memory":"2G"},"requests":{"cpu":1,"memory":"2G"}}` | Resource limits and requests for each inference server instance. You can add necessary GPU request here. | +| inferenceServer.annotations | object | `{}` | Annotations for inference server pods | +| inferenceServer.nodeSelector | object | `{}` | Node selector for inference server pods | +| inferenceServer.tolerations | list | `[]` | Tolerations for inference server pods | +| inferenceServer.affinity | object | `{}` | Affinity rules for inference server pods - another way to request GPUs | +| inferenceServer.modelRepository | object | `{"enabled":false,"mountPath":""}` | Model repository configuration | +| inferenceServer.modelRepository.mountPath | string | `""` | Model repository mount path | +| inferenceServer.service.labels | object | `{}` | | +| inferenceServer.service.annotations | object | `{}` | | +| inferenceServer.service.ports | list | `[{"name":"http","port":8000,"protocol":"TCP","targetPort":8000},{"name":"grpc","port":8001,"protocol":"TCP","targetPort":8001},{"name":"metrics","port":8002,"protocol":"TCP","targetPort":8002}]` | Ports for communication with inference servers | +| inferenceServer.nereid | object | `{"config":{"models":[],"server":{"bind_addr":"[::]:8001","http_addr":"[::]:8002","ml_backends_path":"ml-backends"}},"configPath":"/nereid/nereid.yaml"}` | Nereid server configuration, used only when ``type`` is ``nereid``. Ignored for Triton, which is configured through ``command``/``args`` instead. | +| inferenceServer.nereid.configPath | string | `"/nereid/nereid.yaml"` | Path the rendered ``nereid.yaml`` is mounted at. Nereid reads this file from its working directory, which is ``/nereid`` in the official image. | +| inferenceServer.nereid.config | object | `{"models":[],"server":{"bind_addr":"[::]:8001","http_addr":"[::]:8002","ml_backends_path":"ml-backends"}}` | Contents of ``nereid.yaml``, rendered into a ConfigMap verbatim. ``ml_backends_path`` is resolved relative to the working directory, so use an absolute path when models come from ``modelRepository``. ``bind_addr``/``http_addr`` default to Triton's port numbers so that the Service, Envoy config and ServiceMonitor need no per-server special-casing. | +| inferenceServer.readinessProbe | object | `{"command":["/bin/sh","-c","curl -sf http://localhost:8000/v2/health/ready > /dev/null && [ ! -f /tmp/shutdown ]"],"failureThreshold":10,"initialDelaySeconds":10,"periodSeconds":10,"reset":false,"successThreshold":1,"timeoutSeconds":15}` | Custom readiness probe configuration. The handler is whichever of ``command`` (exec shorthand), ``exec``, ``httpGet`` or ``tcpSocket`` is set; ``reset: true`` drops the probe. | +| inferenceServer.readinessProbe.reset | bool | `false` | If true, will reset settings to k8s defaults (other readinessProbe settings will be ignored) | +| inferenceServer.startupProbe | object | `{"failureThreshold":24,"httpGet":{"path":"/v2/health/ready","port":"http"},"initialDelaySeconds":0,"periodSeconds":10,"reset":false,"timeoutSeconds":15}` | Custom startup probe configuration. Same handler options as ``readinessProbe``. | +| inferenceServer.startupProbe.reset | bool | `false` | If true, will reset settings to k8s defaults (other startupProbe settings will be ignored) | | envoy.enabled | bool | `true` | Enable Envoy Proxy | | envoy.replicas | int | `1` | Number of Envoy Proxy pods in Deployment | | envoy.image | string | `"envoyproxy/envoy:v1.39.1"` | Envoy Proxy Docker image | @@ -59,12 +63,12 @@ | envoy.auth.audiences | list | `[]` | | | envoy.auth.url | string | `""` | | | envoy.auth.port | int | `443` | | -| keda.enabled | bool | `false` | Enable autoscaling (requires Prometheus to also be enabled). Autoscaling will be based on the metric from parameter ``serverLoadMetric``; new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. | -| keda.minReplicaCount | int | `1` | Minimum and maximum number of Triton servers. Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled). With scaleFromZero, a RepositoryIndex request scales Triton to max(1, minReplicaCount), and upgrades keep the live ScaledObject minReplicaCount. | +| keda.enabled | bool | `false` | Enable autoscaling (requires Prometheus to also be enabled). Autoscaling will be based on the metric from parameter ``serverLoadMetric``; new inference servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. | +| keda.minReplicaCount | int | `1` | Minimum and maximum number of inference servers. Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled). With scaleFromZero, a RepositoryIndex request scales the inference server to max(1, minReplicaCount), and upgrades keep the live ScaledObject minReplicaCount. | | keda.maxReplicaCount | int | `2` | | | keda.pollingInterval | int | `30` | How often KEDA polls Prometheus | | keda.cooldownPeriod | int | `120` | Period to wait after the last trigger is inactive before scaling to minReplicaCount | -| keda.zeroIdleReplicas | bool | `false` | If set to true, KEDA sets idleReplicaCount to 0. This cannot scale from 0 back to 1 when the load metric comes from Triton. Use scaleFromZero.enabled with keda.minReplicaCount: 0 instead. | +| keda.zeroIdleReplicas | bool | `false` | If set to true, KEDA sets idleReplicaCount to 0. This cannot scale from 0 back to 1 when the load metric comes from the inference server. Use scaleFromZero.enabled with keda.minReplicaCount: 0 instead. | | keda.scaleUp.stabilizationWindowSeconds | int | `60` | | | keda.scaleUp.periodSeconds | int | `60` | | | keda.scaleUp.stepsize | int | `1` | | diff --git a/docs/configuration-guide.rst b/docs/configuration-guide.rst index 8fe2d7fe..186cc91b 100644 --- a/docs/configuration-guide.rst +++ b/docs/configuration-guide.rst @@ -6,24 +6,95 @@ The full list of parameters can be found in the `Configuration Reference `_. -1. Select a Triton Inference Server Version +1. Select an Inference Server ============================================= +SuperSONIC can run either of two inference servers, selected with +``inferenceServer.type``: + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - ``type`` + - Description + * - ``triton`` (default) + - `NVIDIA Triton Inference Server `_. + Configured through ``inferenceServer.command``/``args``, with models + supplied as a Triton model repository. + * - ``nereid`` + - `Nereid `_, a Rust inference + server. Configured through a ``nereid.yaml`` file rather than flags. + +Both serve the `KServe v2 `_ +inference protocol over gRPC, so clients, Envoy, load balancing, autoscaling +and the scraped metrics are the same either way. Only the image, the model +configuration and the health-probe endpoints differ. + +.. note:: + + Nereid does not export the ``nv_gpu_*`` metrics, so the GPU panels of the + Grafana dashboard and the optional ``metricsCollector`` are Triton-only. + +Selecting Triton +----------------- + - Official versions can be found at `NVIDIA NGC `_. - You can also use custom-built Triton images. -- Refer to the `Nvidia Frameworks Support Matrix `_ +- Refer to the `Nvidia Frameworks Support Matrix `_ for compatibility information (CUDA versions, NVIDIA drivers, etc.). -Triton version must be specified in the ``triton.image`` parameter in the values file. +Triton version must be specified in the ``inferenceServer.image`` parameter in the values file. + +Selecting Nereid +----------------- + +Nereid is configured by a ``nereid.yaml`` file, rendered from +``inferenceServer.nereid.config`` into a ConfigMap and mounted into the +container. Use the image's own entrypoint by setting ``command`` and ``args`` +to ``null``: +.. code-block:: yaml + + inferenceServer: + type: nereid + image: ghcr.io/ngpaladi/nereid-server: + command: null + args: null + nereid: + config: + server: + # Keeping Triton's port numbers means the rest of the chart needs + # no per-server configuration. + bind_addr: "[::]:8001" + http_addr: "[::]:8002" + # Resolved relative to the working directory, so use an absolute + # path when models come from a mounted volume. + ml_backends_path: "/models" + # One entry per model folder; Nereid will not start with an empty list. + models: + - name: mymodel + device: cpu + queue_capacity: 16 + +A complete example is in +`values/values-nereid.yaml `_. + +.. warning:: + + Nereid's Python backend builds a ``venv/`` inside each model folder at load + time, and the container runs as an unprivileged user. A read-only model + volume therefore serves ``.pt``, ONNX and TensorFlow models, but not Python + ones. -2. Configure Triton model repository + +2. Configure the model repository ============================================= - To learn about the structure of model repositories, refer to the `NVIDIA Model Repository Guide `_. -- Model repositories are specified in the ``triton.args`` parameter in the values file. - The parameter contains the full command that launches a Triton server; you can specify +- For Triton, model repositories are specified in the ``inferenceServer.args`` parameter in + the values file. The parameter contains the full command that launches a Triton server; you can specify one or multiple model repositories via the ``--model-repository`` flag. - For example, the following command loads multiple CMS models hosted at CVMFS: @@ -40,8 +111,8 @@ Triton version must be specified in the ``triton.image`` parameter in the values --strict-model-config=false \ --exit-timeout-secs=60 -- Make sure that the model repository paths exist. You can load models from a volume mounted to the Triton container. - The following options for model repository mounting are provided via ``triton.modelRepository`` parameter in ``values.yaml``: +- Make sure that the model repository paths exist. You can load models from a volume mounted to the inference server container. + The following options for model repository mounting are provided via ``inferenceServer.modelRepository`` parameter in ``values.yaml``: .. raw:: html @@ -80,6 +151,13 @@ Triton version must be specified in the ``triton.image`` parameter in the values server: path: + ## -- OR -- + ## Option 5: mount models from a ConfigMap (small models and testing; + ## a ConfigMap holds at most ~1 MiB in total) + storageType: "configMap" + configMap: + name: + .. raw:: html
@@ -89,10 +167,10 @@ Triton version must be specified in the ``triton.image`` parameter in the values

-1. Select Resources for Triton Pods +3. Select Resources for Inference Server Pods ============================================= -- You can configure CPU, memory, and GPU resources for Triton pods via the ``triton.resources`` parameter in the values file: +- You can configure CPU, memory, and GPU resources for inference server pods via the ``inferenceServer.resources`` parameter in the values file: .. code-block:: yaml @@ -106,16 +184,17 @@ Triton version must be specified in the ``triton.image`` parameter in the values cpu: 2 memory: 16G -- In addition, you can use ``triton.nodeSelector``, ``triton.tolerations``, - ``triton.annotations``, and ``triton.affinity`` to steer Triton pods to specific nodes. - This is particularly useful for co-locating Triton pods with Envoy proxy to reduce latency. +- In addition, you can use ``inferenceServer.nodeSelector``, ``inferenceServer.tolerations``, + ``inferenceServer.annotations``, and ``inferenceServer.affinity`` to steer inference server + pods to specific nodes. This is particularly useful for co-locating them with the Envoy + proxy to reduce latency. 4. Configure Envoy Proxy ================================================ By default, Envoy proxy is enabled and configured to provide per-request -load balancing between Triton inference servers. +load balancing between inference servers. Once the SuperSONIC chart is installed, you need an address by which clients can connect to the Envoy proxy and send inference requests. @@ -201,7 +280,7 @@ There are two types of rate limiting available in Envoy Proxy: *listener-level*, This rate limiter can be enabled via the ``envoy.rate_limiter.prometheus_based`` parameter in the values file. - At the moment, this functionality is configured to only reject ``RepositoryIndex`` requests to Triton servers, and it ignores + At the moment, this functionality is configured to only reject ``RepositoryIndex`` requests to inference servers, and it ignores any other requests in order not to slow down the inferences. The metric and threshold for the Prometheus-based rate limiter are the same as those used for the autoscaler (see Prometheus Configuration). @@ -284,12 +363,12 @@ Prometheus is needed to scrape metrics for monitoring, as well as for the rate l Both the rate limiter and the autoscaler are currently configured to use the same Prometheus metric and threshold. They are defined in the ``serverLoadMetric`` and ``serverLoadThreshold`` parameters at the root level of the values file. -The default metric is the inference queue time at the Triton servers, as defined in +The default metric is the inference queue time reported by the inference servers, as defined in `here `_. When the metric value exceeds the threshold, the following happens: -- Autoscaler scales up the number of Triton servers if possible. +- Autoscaler scales up the number of inference servers if possible. - Envoy proxy rejects new ``RepositoryIndex`` requests. The pre-configured Grafana dashboard contains a graph of this metric, entitled "Server Load Metric". @@ -348,7 +427,7 @@ can be enabled via the ``keda.enabled`` parameter in the values file. Please contact cluster administrators if this step of installation fails. The parameters ``keda.minReplicaCount`` and ``keda.maxReplicaCount`` define the range in which -the number of Triton servers can scale. ``keda.pollingInterval`` is how often KEDA queries +the number of inference servers can scale. ``keda.pollingInterval`` is how often KEDA queries Prometheus, and ``keda.cooldownPeriod`` is how long the load metric must stay below the threshold before KEDA scales down to ``minReplicaCount``. @@ -374,14 +453,18 @@ Additional optional parameters can control how quickly the autoscaler reacts to periodSeconds: 30 stepsize: 1 -To keep **zero** Triton replicas when idle, set ``keda.minReplicaCount`` to ``0`` and enable +To keep **zero** inference server replicas when idle, set ``keda.minReplicaCount`` to ``0`` and enable ``scaleFromZero``. Envoy stays running. On a ``RepositoryIndex`` request (the first RPC -used by CMS SONIC clients), SuperSONIC scales Triton to ``max(1, keda.minReplicaCount)`` -replicas and returns the index only after Envoy has a healthy Triton upstream. KEDA then +used by CMS SONIC clients), SuperSONIC scales the inference server to ``max(1, keda.minReplicaCount)`` +replicas and returns the index only after Envoy has a healthy inference server upstream. KEDA then scales up to ``maxReplicaCount`` using the Prometheus load metric. After ``scaleFromZero.holdMinReplicasSeconds`` with no further ``RepositoryIndex`` requests, the ScaledObject minimum returns to ``keda.minReplicaCount``, and KEDA can scale back to zero. +This works with either ``inferenceServer.type``: ``RepositoryIndex`` belongs to the +shared ``inference.GRPCInferenceService`` protocol, so the path Envoy routes on is the +same for both servers. + .. code-block:: yaml envoy: @@ -399,11 +482,11 @@ the ScaledObject minimum returns to ``keda.minReplicaCount``, and KEDA can scale .. warning:: - The client deadline for ``RepositoryIndex`` must cover Triton startup. If no healthy + The client deadline for ``RepositoryIndex`` must cover inference server startup. If no healthy upstream is available within ``scaleFromZero.readyTimeoutSeconds``, the index request is rejected. -``triton.replicas`` is unused when ``scaleFromZero`` is enabled; KEDA owns the replica +``inferenceServer.replicas`` is unused when ``scaleFromZero`` is enabled; KEDA owns the replica count. Helm upgrades keep the live ScaledObject ``minReplicaCount`` so they do not interrupt an active hold. The hold deadline is stored as an annotation on the ScaledObject, so the admission sidecars of multiple Envoy replicas share one hold @@ -412,7 +495,7 @@ and none can release a peer's active hold. ``scaleFromZero`` requires ``keda.ena Do not set ``keda.zeroIdleReplicas: true`` together with ``minReplicaCount: 0``. ``zeroIdleReplicas`` sets KEDA ``idleReplicaCount`` to 0 and cannot scale from 0 back to 1 -when the load metric is scraped from Triton. Use ``scaleFromZero`` for that. +when the load metric is scraped from the inference server. Use ``scaleFromZero`` for that. An example is ``values/values-geddes-cms.yaml``. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 27e1cb03..3edb63c2 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -9,7 +9,7 @@ Pre-requisites 2. `Helm `_ 3. Access to an existing `Prometheus `_ instance in the cluster, or sufficient permissions to deploy a custom instance (preferred). 4. If using autoscaling, you may need to ask cluster administrators to install CustomResourceDefinitions for `KEDA `_. - To keep zero Triton replicas when idle, enable ``scaleFromZero`` as described in the + To keep zero inference server replicas when idle, enable ``scaleFromZero`` as described in the :doc:`configuration guide `. Installation diff --git a/helm/supersonic/README.md b/helm/supersonic/README.md index 45a9fe60..5c1d7118 100644 --- a/helm/supersonic/README.md +++ b/helm/supersonic/README.md @@ -18,6 +18,7 @@ applications in large high energy physics (HEP) and multi-messenger astrophysics Currently, SuperSONIC supports the following functionality: - GPU inference-as-a-service via [Nvidia Triton Inference Server](https://developer.nvidia.com/triton-inference-server) + or [Nereid](https://github.com/ngpaladi/nereid-server), selected with `inferenceServer.type` - Load balancing across many GPUs via [Envoy Proxy](envoyproxy.io) - Load-based autoscaling via [KEDA](keda.sh), including scale from zero replicas on `RepositoryIndex` - Monitoring via [Prometheus](https://prometheus.io) and [Grafana](https://grafana.com) @@ -55,7 +56,7 @@ Currently, SuperSONIC supports the following functionality: helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update kubectl create namespace monitoring - helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + helm install prometheus-operator prometheus-community/kube-prometheus-stack --version 89.2.4 --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false ``` - [KEDA](https://keda.sh) CRDs (only if using autoscaling) @@ -116,7 +117,7 @@ kubectl apply -f cvmfs/cvmfs-storageclass.yaml -n cvmfs-csi
2. Install SuperSONIC with minimal configuration -The minimal deployment will install only a single CPU-based Triton server and an Envoy Proxy. +The minimal deployment will install only a single CPU-based inference server (Triton by default) and an Envoy Proxy. We will use [`values/values-minimal.yaml`](values/values-minimal.yaml) as our minimal configuration file. diff --git a/helm/supersonic/cfg/admission-server.py b/helm/supersonic/cfg/admission-server.py index bb04d876..549fbc9b 100644 --- a/helm/supersonic/cfg/admission-server.py +++ b/helm/supersonic/cfg/admission-server.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Scale Triton to at least max(1, minReplicaCount) replicas on /wake.""" +"""Scale the inference server to at least max(1, minReplicaCount) replicas on /wake.""" import json import os @@ -16,7 +16,7 @@ NS_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" HOLD_TTL = int(os.environ.get("HOLD_MIN_REPLICAS_SECONDS", "300")) -TRITON_DEPLOYMENT = os.environ.get("GPU_DEPLOYMENT", "") +INFERENCE_SERVER_DEPLOYMENT = os.environ.get("INFERENCE_SERVER_DEPLOYMENT", "") SCALEDOBJECT_NAME = os.environ.get("SCALEDOBJECT_NAME", "") IDLE_MIN_REPLICAS = int(os.environ.get("IDLE_MIN_REPLICAS", "0")) # On wake, scale to the configured minReplicaCount, but never below one replica. @@ -131,8 +131,8 @@ def _hold_until(scaled_object): return None -def ensure_triton_replica(extend_hold=False): - """Raise the ScaledObject minimum and the Triton Deployment to the wake target. +def ensure_inference_server_replica(extend_hold=False): + """Raise the ScaledObject minimum and the inference server Deployment to the wake target. With extend_hold, also advance the shared hold annotation to now + HOLD_TTL in the same patch (it is only ever moved forward).""" @@ -159,17 +159,17 @@ def ensure_triton_replica(extend_hold=False): return False _scaled_object_held = True path = ( - f"/apis/apps/v1/namespaces/{_namespace()}/deployments/{TRITON_DEPLOYMENT}/scale" + f"/apis/apps/v1/namespaces/{_namespace()}/deployments/{INFERENCE_SERVER_DEPLOYMENT}/scale" ) try: current = _api_request("GET", path) replicas = _spec_int(current, "replicas") if replicas < WAKE_MIN_REPLICAS: _api_request("PATCH", path, {"spec": {"replicas": WAKE_MIN_REPLICAS}}) - _log(f"scaled {TRITON_DEPLOYMENT} {replicas} -> {WAKE_MIN_REPLICAS}") + _log(f"scaled {INFERENCE_SERVER_DEPLOYMENT} {replicas} -> {WAKE_MIN_REPLICAS}") return True except ApiError as exc: - _log(f"scale {TRITON_DEPLOYMENT}: {exc}") + _log(f"scale {INFERENCE_SERVER_DEPLOYMENT}: {exc}") return False @@ -225,7 +225,7 @@ def wake(): # scaled up and extended the hold; do not queue another identical pass. if _last_wake_pass >= arrived: return True - ok = ensure_triton_replica(extend_hold=True) + ok = ensure_inference_server_replica(extend_hold=True) if ok: _last_wake_pass = time.time() return ok @@ -240,7 +240,7 @@ def _watch_loop(): if _hold_active(): if now - last_reassert >= 2: with _scale_lock: - ensure_triton_replica() + ensure_inference_server_replica() last_reassert = now elif _scaled_object_held: with _scale_lock: @@ -302,9 +302,9 @@ def do_GET(self): def main(): - if not TRITON_DEPLOYMENT or not SCALEDOBJECT_NAME: + if not INFERENCE_SERVER_DEPLOYMENT or not SCALEDOBJECT_NAME: raise SystemExit( - "admission: GPU_DEPLOYMENT and SCALEDOBJECT_NAME are required" + "admission: INFERENCE_SERVER_DEPLOYMENT and SCALEDOBJECT_NAME are required" ) try: _namespace() @@ -315,7 +315,7 @@ def main(): watcher = threading.Thread(target=_watch_loop, name="scale-watch", daemon=True) watcher.start() server = ThreadingHTTPServer((LISTEN_HOST, LISTEN_PORT), Handler) - _log(f"listening on {LISTEN_HOST}:{LISTEN_PORT} deployment={TRITON_DEPLOYMENT}") + _log(f"listening on {LISTEN_HOST}:{LISTEN_PORT} deployment={INFERENCE_SERVER_DEPLOYMENT}") server.serve_forever() diff --git a/helm/supersonic/cfg/envoy-filter.lua b/helm/supersonic/cfg/envoy-filter.lua index 0e052f26..5d585770 100644 --- a/helm/supersonic/cfg/envoy-filter.lua +++ b/helm/supersonic/cfg/envoy-filter.lua @@ -12,19 +12,19 @@ function envoy_on_request(request_handle) local scale_from_zero = ("SCALE_FROM_ZERO_ENABLED" == "true") local prometheus_rate_limit_enabled = ("PROMETHEUS_RATE_LIMIT_ENABLED" == "true") - -- Scale Triton to at least one replica and wait for a healthy Envoy upstream before forwarding RepositoryIndex. + -- Scale the inference server to at least one replica and wait for a healthy Envoy upstream before forwarding RepositoryIndex. if scale_from_zero then local timeout_seconds = tonumber("READY_TIMEOUT_SECONDS") or 300 - request_handle:logInfo("Scale-from-zero: starting GPU Triton") + request_handle:logInfo("Scale-from-zero: starting the inference server") -- /wake may wait behind an in-flight scaling pass and then make up to -- four Kubernetes API calls of its own (3s timeout each), so give it -- enough headroom for a slow apiserver. local wake_headers = request_handle:httpCall( - "triton_admission", + "inference_server_admission", { [":method"] = "GET", [":path"] = "/wake", - [":authority"] = "triton_admission" + [":authority"] = "inference_server_admission" }, "", 30000 @@ -35,7 +35,7 @@ function envoy_on_request(request_handle) return end - -- Wait until Envoy reports a healthy Triton host, or until the deadline passes. + -- Wait until Envoy reports a healthy inference server host, or until the deadline passes. -- Envoy Lua has no sleep primitive, so the loop is paced by the admission -- sidecar's /sleep endpoint, which blocks for ~1s before responding. local healthy = false @@ -46,7 +46,7 @@ function envoy_on_request(request_handle) "envoy_admin", { [":method"] = "GET", - [":path"] = "/stats?filter=cluster.triton_grpc_service.membership_healthy", + [":path"] = "/stats?filter=cluster.inference_server_grpc_service.membership_healthy", [":authority"] = "envoy_admin" }, "", @@ -54,18 +54,18 @@ function envoy_on_request(request_handle) ) local n = 0 if stats_body then - n = tonumber(string.match(stats_body, "cluster%.triton_grpc_service%.membership_healthy: ([0-9]+)")) or 0 + n = tonumber(string.match(stats_body, "cluster%.inference_server_grpc_service%.membership_healthy: ([0-9]+)")) or 0 end if n > 0 then healthy = true break end local sleep_headers = request_handle:httpCall( - "triton_admission", + "inference_server_admission", { [":method"] = "GET", [":path"] = "/sleep", - [":authority"] = "triton_admission" + [":authority"] = "inference_server_admission" }, "", 2000 @@ -82,13 +82,13 @@ function envoy_on_request(request_handle) end end if not healthy then - request_handle:logErr("No healthy Triton upstream in time; rejecting RepositoryIndex") - request_handle:streamInfo():dynamicMetadata():set("envoy.lua", "reject_reason", "no healthy upstream: Triton did not become ready in time") + request_handle:logErr("No healthy inference server upstream in time; rejecting RepositoryIndex") + request_handle:streamInfo():dynamicMetadata():set("envoy.lua", "reject_reason", "no healthy upstream: inference server did not become ready in time") return end - request_handle:logInfo("GPU Triton has a healthy Envoy upstream") + request_handle:logInfo("Inference server has a healthy Envoy upstream") - -- Refresh the KEDA hold now that Triton is ready. The first /wake + -- Refresh the KEDA hold now that the inference server is ready. The first /wake -- anchored hold-until at the moment the wake started, so by the time -- the index is answered only hold - startup_time of it would remain -- (and could be nearly nothing after a slow start). /wake only moves @@ -99,11 +99,11 @@ function envoy_on_request(request_handle) -- admission sidecar finishes the wake pass server-side even if Envoy -- stops waiting for the reply. local refresh_headers = request_handle:httpCall( - "triton_admission", + "inference_server_admission", { [":method"] = "GET", [":path"] = "/wake", - [":authority"] = "triton_admission" + [":authority"] = "inference_server_admission" }, "", 10000 diff --git a/helm/supersonic/dashboards/default.json b/helm/supersonic/dashboards/default.json index 1a5fc438..c26b1517 100644 --- a/helm/supersonic/dashboards/default.json +++ b/helm/supersonic/dashboards/default.json @@ -128,7 +128,7 @@ "useBackend": false } ], - "title": "Number of Triton Servers", + "title": "Number of Inference Servers", "type": "stat" }, { @@ -136,7 +136,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Latency components measured by Triton servers, as well as total latency measured at proxy.", + "description": "Latency components measured by the inference servers, as well as total latency measured at proxy.", "fieldConfig": { "defaults": { "color": { @@ -502,7 +502,7 @@ "useBackend": false } ], - "title": "Number of Triton Servers", + "title": "Number of Inference Servers", "type": "timeseries" }, { diff --git a/helm/supersonic/templates/NOTES.txt b/helm/supersonic/templates/NOTES.txt index dfa0b48d..0bb7bd08 100644 --- a/helm/supersonic/templates/NOTES.txt +++ b/helm/supersonic/templates/NOTES.txt @@ -1,5 +1,7 @@ --- {{- /* Run validation checks */ -}} +{{- include "supersonic.validateNoLegacyTritonValues" . -}} +{{- include "supersonic.validateInferenceServerType" . -}} {{- include "supersonic.validateGrafanaAddressConsistency" . -}} {{- include "supersonic.validateGrafanaValues" . -}} {{- include "supersonic.validatePrometheusAddressConsistency" . -}} diff --git a/helm/supersonic/templates/_helpers/_helpers.tpl b/helm/supersonic/templates/_helpers/_helpers.tpl index 38d50514..7232ecb7 100644 --- a/helm/supersonic/templates/_helpers/_helpers.tpl +++ b/helm/supersonic/templates/_helpers/_helpers.tpl @@ -12,10 +12,10 @@ Get instance name (equal to release name unless overridden) {{- end -}} {{/* -Get Triton server name +Get inference server name */}} -{{- define "supersonic.tritonName" -}} -{{- printf "%s-triton" (include "supersonic.name" .) | trunc 63 | trimSuffix "-" -}} +{{- define "supersonic.inferenceServerName" -}} +{{- printf "%s-inference-server" (include "supersonic.name" .) | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* @@ -74,3 +74,14 @@ Get gRPC endpoint for client connections {{- end -}} {{- end -}} {{- end -}} + +{{/* +Fail on the pre-0.4.0 `triton:` values block, which was renamed to +`inferenceServer:`. Helm would otherwise ignore the stale key and silently +deploy chart defaults, so an explicit error is the kinder failure. +*/}} +{{- define "supersonic.validateNoLegacyTritonValues" -}} +{{- if .Values.triton -}} +{{- fail "The `triton:` values block was renamed to `inferenceServer:`.\nRename the top-level `triton:` key in your values file (its contents are unchanged).\nSee https://fastmachinelearning.org/SuperSONIC for the migration notes." -}} +{{- end -}} +{{- end -}} diff --git a/helm/supersonic/templates/_helpers/_inference-server.tpl b/helm/supersonic/templates/_helpers/_inference-server.tpl new file mode 100644 index 00000000..1e81804c --- /dev/null +++ b/helm/supersonic/templates/_helpers/_inference-server.tpl @@ -0,0 +1,77 @@ +{{- /* templates/_helpers/_inference-server.tpl */ -}} + +{{/* +Inference server implementation, defaulting to Triton. +*/}} +{{- define "supersonic.inferenceServerType" -}} +{{- .Values.inferenceServer.type | default "triton" -}} +{{- end -}} + +{{/* +True when the configured inference server is Nereid. +*/}} +{{- define "supersonic.nereidEnabled" -}} +{{- if eq (include "supersonic.inferenceServerType" .) "nereid" -}}true{{- end -}} +{{- end -}} + +{{/* +Reject an unknown inferenceServer.type before it produces a confusing +half-configured Deployment. +*/}} +{{- define "supersonic.validateInferenceServerType" -}} +{{- $type := include "supersonic.inferenceServerType" . -}} +{{- if not (has $type (list "triton" "nereid")) -}} +{{- fail (printf "Unknown inferenceServer.type %q. Supported values: triton, nereid." $type) -}} +{{- end -}} +{{- end -}} + +{{/* +Render one probe body from a values block. + +The handler is whichever one of these the block sets: + command: exec shorthand, a bare command list + exec: full exec handler + httpGet: HTTP handler + tcpSocket: TCP handler + +Exactly one may be set. Helm deep-merges values, so a values file that adds a +handler does *not* drop the one this chart defaults to -- switching handler +type means nulling the inherited key (`command: null`). Setting two is +therefore almost always that mistake rather than an intent, and picking one by +precedence would silently probe the wrong endpoint, so it fails instead. + +Triton is probed over its HTTP port; Nereid's image ships no curl, so it uses +httpGet against the KServe v2 health endpoints on its own HTTP port. + +Usage: include "supersonic.inferenceServerProbe" .Values.inferenceServer.readinessProbe +*/}} +{{- define "supersonic.inferenceServerProbe" -}} +{{- $probe := . -}} +{{- $set := list -}} +{{- range $handler := (list "command" "exec" "httpGet" "tcpSocket") -}} +{{- if index $probe $handler -}} +{{- $set = append $set $handler -}} +{{- end -}} +{{- end -}} +{{- if gt (len $set) 1 -}} +{{- fail (printf "A probe sets more than one handler (%s). Set exactly one; because Helm merges values with the chart defaults, switching handler type means nulling the inherited key, e.g. `command: null`." (join ", " $set)) -}} +{{- end -}} +{{- if $probe.command }} +exec: + command: {{ toYaml $probe.command | nindent 4 }} +{{- else if $probe.exec }} +exec: + {{- toYaml $probe.exec | nindent 2 }} +{{- else if $probe.httpGet }} +httpGet: + {{- toYaml $probe.httpGet | nindent 2 }} +{{- else if $probe.tcpSocket }} +tcpSocket: + {{- toYaml $probe.tcpSocket | nindent 2 }} +{{- end }} +{{- range $field := (list "initialDelaySeconds" "periodSeconds" "timeoutSeconds" "successThreshold" "failureThreshold") }} +{{- if hasKey $probe $field }} +{{ $field }}: {{ index $probe $field }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/helm/supersonic/templates/cvmfs-pvc.yaml b/helm/supersonic/templates/cvmfs-pvc.yaml index 1254a361..210bad94 100644 --- a/helm/supersonic/templates/cvmfs-pvc.yaml +++ b/helm/supersonic/templates/cvmfs-pvc.yaml @@ -1,4 +1,4 @@ -{{ if eq .Values.triton.modelRepository.storageType "cvmfs-pvc" }} +{{ if eq .Values.inferenceServer.modelRepository.storageType "cvmfs-pvc" }} {{- $mypvc := (lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "cvmfs")) }} {{- if not $mypvc }} apiVersion: v1 diff --git a/helm/supersonic/templates/envoy/admission.yaml b/helm/supersonic/templates/envoy/admission.yaml index 03b7860b..5383d0cc 100644 --- a/helm/supersonic/templates/envoy/admission.yaml +++ b/helm/supersonic/templates/envoy/admission.yaml @@ -28,7 +28,7 @@ metadata: rules: - apiGroups: ["apps"] resources: ["deployments/scale"] - resourceNames: [{{ include "supersonic.tritonName" . | quote }}] + resourceNames: [{{ include "supersonic.inferenceServerName" . | quote }}] verbs: ["get", "patch", "update"] - apiGroups: ["keda.sh"] resources: ["scaledobjects"] diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index fa74e4fb..eb2a44c9 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -59,9 +59,9 @@ static_resources: - match: path: "/inference.GRPCInferenceService/RepositoryIndex" route: - cluster: triton_grpc_service + cluster: inference_server_grpc_service timeout: {{ .envoy.grpc_route_timeout }} - # The Lua filter holds this stream while Triton scales up from zero. + # The Lua filter holds this stream while the inference server scales up from zero. # RouteAction.idle_timeout overrides the HCM stream_idle_timeout (default # 300s) for this route only and is applied at route refresh in # decodeHeaders, i.e. before the HTTP filter chain runs, so it covers the @@ -85,7 +85,7 @@ static_resources: - match: prefix: "/" route: - cluster: triton_grpc_service + cluster: inference_server_grpc_service timeout: {{ .envoy.grpc_route_timeout }} http_filters: {{- if or .envoy.rate_limiter.prometheus_based.enabled .scaleFromZeroEnabled }} @@ -169,11 +169,11 @@ static_resources: {{- end }} {{- end }} {{- if .scaleFromZeroEnabled }} - - name: triton_admission + - name: inference_server_admission type: STATIC connect_timeout: 1s load_assignment: - cluster_name: triton_admission + cluster_name: inference_server_admission endpoints: - lb_endpoints: - endpoint: @@ -195,7 +195,7 @@ static_resources: port_value: {{ .envoyAdminPort }} {{- end }} - - name: triton_grpc_service + - name: inference_server_grpc_service connect_timeout: {{ if .scaleFromZeroEnabled }}2s{{ else }}0.25s{{ end }} type: STRICT_DNS {{- if .scaleFromZeroEnabled }} @@ -211,14 +211,14 @@ static_resources: dns_lookup_family: V4_ONLY {{- include "envoy.http2ProtocolOptions" . | nindent 6 }} load_assignment: - cluster_name: triton_grpc_service + cluster_name: inference_server_grpc_service endpoints: - lb_endpoints: - endpoint: address: socket_address: - address: {{ .tritonName }}.{{ .root.Release.Namespace }}.svc.cluster.local. - port_value: {{ .tritonGrpcPort }} + address: {{ .inferenceServerName }}.{{ .root.Release.Namespace }}.svc.cluster.local. + port_value: {{ .inferenceServerGrpcPort }} {{- end }} {{- end }} @@ -231,12 +231,12 @@ static_resources: {{- if not .Values.envoy.external_config.load_from_configmap }} {{- /* Only prepare dynamic configuration context when not using external config */}} {{- /* Define variables for ports */}} -{{- $tritonGrpcPort := "" -}} +{{- $inferenceServerGrpcPort := "" -}} {{- $envoyGrpcPort := "" -}} {{- $envoyAdminPort := "" -}} -{{- range .Values.triton.service.ports }} +{{- range .Values.inferenceServer.service.ports }} {{- if eq .name "grpc" }} - {{- $tritonGrpcPort = .port }} + {{- $inferenceServerGrpcPort = .port }} {{- end }} {{- end }} {{- range .Values.envoy.service.ports }} @@ -247,17 +247,17 @@ static_resources: {{- $envoyAdminPort = .port }} {{- end }} {{- end }} -{{- $tritonName := include "supersonic.tritonName" . }} +{{- $inferenceServerName := include "supersonic.inferenceServerName" . }} {{- /* Create a context to pass to the template */}} {{- $envoyContext = dict "envoyAdminPort" $envoyAdminPort "envoyGrpcPort" $envoyGrpcPort - "tritonGrpcPort" $tritonGrpcPort + "inferenceServerGrpcPort" $inferenceServerGrpcPort "prometheus" .Values.prometheus "envoy" .Values.envoy - "triton" .Values.triton - "tritonName" $tritonName + "inferenceServer" .Values.inferenceServer + "inferenceServerName" $inferenceServerName "scaleFromZeroEnabled" (eq (include "supersonic.scaleFromZeroEnabled" .) "true") "root" . }} diff --git a/helm/supersonic/templates/envoy/deployment.yaml b/helm/supersonic/templates/envoy/deployment.yaml index 8afdea37..792e2a5f 100644 --- a/helm/supersonic/templates/envoy/deployment.yaml +++ b/helm/supersonic/templates/envoy/deployment.yaml @@ -69,8 +69,8 @@ spec: env: - name: HOLD_MIN_REPLICAS_SECONDS value: {{ $sfz.holdMinReplicasSeconds | quote }} - - name: GPU_DEPLOYMENT - value: {{ include "supersonic.tritonName" . | quote }} + - name: INFERENCE_SERVER_DEPLOYMENT + value: {{ include "supersonic.inferenceServerName" . | quote }} - name: SCALEDOBJECT_NAME value: {{ printf "%s-keda-so" (include "supersonic.name" .) | quote }} - name: IDLE_MIN_REPLICAS diff --git a/helm/supersonic/templates/inference-server/deployment.yaml b/helm/supersonic/templates/inference-server/deployment.yaml new file mode 100644 index 00000000..a672c6fc --- /dev/null +++ b/helm/supersonic/templates/inference-server/deployment.yaml @@ -0,0 +1,120 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "supersonic.inferenceServerName" . }} + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ include "supersonic.name" . }} + app.kubernetes.io/component: inference-server +spec: +{{- if ne (include "supersonic.scaleFromZeroEnabled" .) "true" }} + replicas: {{ .Values.inferenceServer.replicas | default 1 }} +{{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ include "supersonic.name" . }} + app.kubernetes.io/component: inference-server + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ include "supersonic.name" . }} + app.kubernetes.io/component: inference-server + {{- with .Values.inferenceServer.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + terminationGracePeriodSeconds: 60 + containers: + - name: inference-server + image: {{ .Values.inferenceServer.image }} + resources: + {{ toYaml .Values.inferenceServer.resources | nindent 10 }} + ports: + {{- range .Values.inferenceServer.service.ports }} + - name: {{ .name }} + containerPort: {{ .port }} + protocol: {{ .protocol }} + {{- end }} + volumeMounts: + {{- if .Values.inferenceServer.modelRepository.enabled }} + - name: model-repository + mountPath: {{ .Values.inferenceServer.modelRepository.mountPath }} + {{- /* mountPropagation is meaningful for host-backed volumes only, + and the API server rejects it on a configMap volume. */}} + {{- if ne .Values.inferenceServer.modelRepository.storageType "configMap" }} + mountPropagation: HostToContainer + {{- end }} + readOnly: true + {{- end }} + {{- if eq (include "supersonic.nereidEnabled" .) "true" }} + {{- /* subPath so the mount does not shadow the model folders that + also live under Nereid's working directory */}} + - name: nereid-config + mountPath: {{ .Values.inferenceServer.nereid.configPath }} + subPath: nereid.yaml + readOnly: true + {{- end }} + {{- /* Omitted rather than emitted empty, so a server configured through + a file (Nereid) can fall back to its image's own entrypoint. */}} + {{- with .Values.inferenceServer.command }} + command: {{ toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.inferenceServer.args }} + args: {{ toYaml . | nindent 10 }} + {{- end }} +{{- if not (eq .Values.inferenceServer.startupProbe.reset true) }} + startupProbe: + {{- include "supersonic.inferenceServerProbe" .Values.inferenceServer.startupProbe | nindent 10 }} +{{- end }} +{{- if not (eq .Values.inferenceServer.readinessProbe.reset true) }} + readinessProbe: + {{- include "supersonic.inferenceServerProbe" .Values.inferenceServer.readinessProbe | nindent 10 }} +{{- end }} + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "touch", "/tmp/shutdown;", "sleep 60"] + volumes: + {{- if eq (include "supersonic.nereidEnabled" .) "true" }} + - name: nereid-config + configMap: + name: {{ include "supersonic.inferenceServerName" . }}-nereid-config + {{- end }} + {{- if .Values.inferenceServer.modelRepository.enabled }} + - name: model-repository + {{- if eq .Values.inferenceServer.modelRepository.storageType "nfs" }} + nfs: + server: {{ .Values.inferenceServer.modelRepository.nfs.server }} + path: {{ .Values.inferenceServer.modelRepository.nfs.path }} + {{- else if eq .Values.inferenceServer.modelRepository.storageType "pvc" }} + persistentVolumeClaim: + claimName: {{ .Values.inferenceServer.modelRepository.pvc.claimName }} + {{- else if eq .Values.inferenceServer.modelRepository.storageType "cvmfs-hostPath" }} + hostPath: + path: /cvmfs + {{- else if eq .Values.inferenceServer.modelRepository.storageType "cvmfs-pvc" }} + persistentVolumeClaim: + claimName: cvmfs + readOnly: true + {{- else if eq .Values.inferenceServer.modelRepository.storageType "configMap" }} + configMap: + name: {{ .Values.inferenceServer.modelRepository.configMap.name }} + {{- end }} + {{- end }} + +{{- if .Values.inferenceServer.affinity }} + affinity: {{ toYaml .Values.inferenceServer.affinity | nindent 8}} +{{- end }} + +{{- if .Values.inferenceServer.nodeSelector }} + nodeSelector: +{{ toYaml .Values.inferenceServer.nodeSelector | nindent 8 }} +{{- end }} +{{- if .Values.inferenceServer.tolerations }} + tolerations: +{{ toYaml .Values.inferenceServer.tolerations | nindent 8 }} +{{- end }} + restartPolicy: Always \ No newline at end of file diff --git a/helm/supersonic/templates/inference-server/nereid-config.yaml b/helm/supersonic/templates/inference-server/nereid-config.yaml new file mode 100644 index 00000000..d300936e --- /dev/null +++ b/helm/supersonic/templates/inference-server/nereid-config.yaml @@ -0,0 +1,16 @@ +{{- if eq (include "supersonic.nereidEnabled" .) "true" }} +{{- /* Nereid reads nereid.yaml from its working directory and resolves + server.ml_backends_path relative to it, so point ml_backends_path at + the model repository mount path when models come from a volume. */}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "supersonic.inferenceServerName" . }}-nereid-config + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + app.kubernetes.io/instance: {{ include "supersonic.name" . }} + app.kubernetes.io/component: inference-server +data: + nereid.yaml: |- + {{- toYaml .Values.inferenceServer.nereid.config | nindent 4 }} +{{- end }} diff --git a/helm/supersonic/templates/triton/service.yaml b/helm/supersonic/templates/inference-server/service.yaml similarity index 51% rename from helm/supersonic/templates/triton/service.yaml rename to helm/supersonic/templates/inference-server/service.yaml index 57a4879c..5d543487 100644 --- a/helm/supersonic/templates/triton/service.yaml +++ b/helm/supersonic/templates/inference-server/service.yaml @@ -1,22 +1,22 @@ apiVersion: v1 kind: Service metadata: - name: {{ include "supersonic.tritonName" . }} + name: {{ include "supersonic.inferenceServerName" . }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton - {{- if .Values.triton.service.labels }} -{{ toYaml .Values.triton.service.labels | nindent 4 }} + app.kubernetes.io/component: inference-server + {{- if .Values.inferenceServer.service.labels }} +{{ toYaml .Values.inferenceServer.service.labels | nindent 4 }} {{- end }} annotations: - {{- if .Values.triton.service.annotations }} -{{ toYaml .Values.triton.service.annotations | nindent 4 }} + {{- if .Values.inferenceServer.service.annotations }} +{{ toYaml .Values.inferenceServer.service.annotations | nindent 4 }} {{- end }} spec: clusterIP: None ports: - {{- range .Values.triton.service.ports }} + {{- range .Values.inferenceServer.service.ports }} - name: {{ .name }} port: {{ .port }} targetPort: {{ .targetPort }} @@ -25,4 +25,4 @@ spec: selector: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton \ No newline at end of file + app.kubernetes.io/component: inference-server \ No newline at end of file diff --git a/helm/supersonic/templates/keda/so.yaml b/helm/supersonic/templates/keda/so.yaml index 9d412962..fa3c7ed2 100644 --- a/helm/supersonic/templates/keda/so.yaml +++ b/helm/supersonic/templates/keda/so.yaml @@ -20,7 +20,7 @@ metadata: app.kubernetes.io/component: keda spec: scaleTargetRef: - name: {{ include "supersonic.tritonName" . }} + name: {{ include "supersonic.inferenceServerName" . }} kind: Deployment pollingInterval: {{ .Values.keda.pollingInterval | default 30 }} cooldownPeriod: {{ .Values.keda.cooldownPeriod | default 120 }} diff --git a/helm/supersonic/templates/monitoring/prometheus-configmap.yaml b/helm/supersonic/templates/monitoring/prometheus-configmap.yaml index 703534aa..556e29d7 100644 --- a/helm/supersonic/templates/monitoring/prometheus-configmap.yaml +++ b/helm/supersonic/templates/monitoring/prometheus-configmap.yaml @@ -14,7 +14,7 @@ data: evaluation_interval: {{ .Values.prometheus.server.global.evaluation_interval }} scrape_configs: - - job_name: "{{ include "supersonic.tritonName" . }}" + - job_name: "{{ include "supersonic.inferenceServerName" . }}" kubernetes_sd_configs: - role: pod namespaces: @@ -23,7 +23,7 @@ data: metrics_path: /metrics relabel_configs: - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_component] - regex: "triton" + regex: "inference-server" action: keep - source_labels: [__meta_kubernetes_pod_container_port_number] action: keep diff --git a/helm/supersonic/templates/monitoring/servicemonitor.yaml b/helm/supersonic/templates/monitoring/servicemonitor.yaml index 7882c71a..6849b2d5 100644 --- a/helm/supersonic/templates/monitoring/servicemonitor.yaml +++ b/helm/supersonic/templates/monitoring/servicemonitor.yaml @@ -2,18 +2,18 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: {{ include "supersonic.name" . }}-triton-monitor + name: {{ include "supersonic.name" . }}-inference-server-monitor namespace: {{ .Release.Namespace }} labels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton + app.kubernetes.io/component: inference-server spec: selector: matchLabels: app.kubernetes.io/name: {{ .Chart.Name }} app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton + app.kubernetes.io/component: inference-server endpoints: - port: metrics path: /metrics diff --git a/helm/supersonic/templates/triton/deployment.yaml b/helm/supersonic/templates/triton/deployment.yaml deleted file mode 100644 index 604da685..00000000 --- a/helm/supersonic/templates/triton/deployment.yaml +++ /dev/null @@ -1,106 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "supersonic.tritonName" . }} - labels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton -spec: -{{- if ne (include "supersonic.scaleFromZeroEnabled" .) "true" }} - replicas: {{ .Values.triton.replicas | default 1 }} -{{- end }} - selector: - matchLabels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton - template: - metadata: - labels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ include "supersonic.name" . }} - app.kubernetes.io/component: triton - {{- with .Values.triton.annotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - terminationGracePeriodSeconds: 60 - containers: - - name: triton-server - image: {{ .Values.triton.image }} - resources: - {{ toYaml .Values.triton.resources | nindent 10 }} - ports: - {{- range .Values.triton.service.ports }} - - name: {{ .name }} - containerPort: {{ .port }} - protocol: {{ .protocol }} - {{- end }} - volumeMounts: - {{- if .Values.triton.modelRepository.enabled }} - - name: model-repository - mountPath: {{ .Values.triton.modelRepository.mountPath }} - mountPropagation: HostToContainer - readOnly: true - {{- end }} - command: {{ toYaml .Values.triton.command | nindent 10 }} - args: {{ toYaml .Values.triton.args | nindent 10 }} -{{- if not (eq .Values.triton.startupProbe.reset true) }} - startupProbe: - initialDelaySeconds: {{ .Values.triton.startupProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.triton.startupProbe.periodSeconds }} - failureThreshold: {{ .Values.triton.startupProbe.failureThreshold }} - timeoutSeconds: {{ .Values.triton.startupProbe.timeoutSeconds | default 1 }} - httpGet: - path: {{ .Values.triton.startupProbe.httpGet.path }} - port: {{ .Values.triton.startupProbe.httpGet.port }} -{{- end }} -{{- if not (eq .Values.triton.readinessProbe.reset true) }} - readinessProbe: - exec: - command: {{ toYaml .Values.triton.readinessProbe.command | nindent 12 }} - initialDelaySeconds: {{ .Values.triton.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.triton.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.triton.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.triton.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.triton.readinessProbe.failureThreshold }} -{{- end }} - lifecycle: - preStop: - exec: - command: ["/bin/sh", "-c", "touch", "/tmp/shutdown;", "sleep 60"] - volumes: - {{- if .Values.triton.modelRepository.enabled }} - - name: model-repository - {{- if eq .Values.triton.modelRepository.storageType "nfs" }} - nfs: - server: {{ .Values.triton.modelRepository.nfs.server }} - path: {{ .Values.triton.modelRepository.nfs.path }} - {{- else if eq .Values.triton.modelRepository.storageType "pvc" }} - persistentVolumeClaim: - claimName: {{ .Values.triton.modelRepository.pvc.claimName }} - {{- else if eq .Values.triton.modelRepository.storageType "cvmfs-hostPath" }} - hostPath: - path: /cvmfs - {{- else if eq .Values.triton.modelRepository.storageType "cvmfs-pvc" }} - persistentVolumeClaim: - claimName: cvmfs - readOnly: true - {{- end }} - {{- end }} - -{{- if .Values.triton.affinity }} - affinity: {{ toYaml .Values.triton.affinity | nindent 8}} -{{- end }} - -{{- if .Values.triton.nodeSelector }} - nodeSelector: -{{ toYaml .Values.triton.nodeSelector | nindent 8 }} -{{- end }} -{{- if .Values.triton.tolerations }} - tolerations: -{{ toYaml .Values.triton.tolerations | nindent 8 }} -{{- end }} - restartPolicy: Always \ No newline at end of file diff --git a/helm/supersonic/values.schema.json b/helm/supersonic/values.schema.json index 3d64c1d7..9053870b 100644 --- a/helm/supersonic/values.schema.json +++ b/helm/supersonic/values.schema.json @@ -26,17 +26,14 @@ "admissionImage": { "type": "string" } - }, - "required": [ - "admissionImage", - "enabled", - "holdMinReplicasSeconds", - "readyTimeoutSeconds" - ] + } }, - "triton": { + "inferenceServer": { "type": "object", "properties": { + "type": { + "type": "string" + }, "replicas": { "type": "integer" }, @@ -67,11 +64,7 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } }, "requests": { "type": "object", @@ -82,17 +75,9 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } } - }, - "required": [ - "limits", - "requests" - ] + } }, "annotations": { "type": "object" @@ -115,11 +100,7 @@ "mountPath": { "type": "string" } - }, - "required": [ - "enabled", - "mountPath" - ] + } }, "service": { "type": "object", @@ -147,21 +128,40 @@ "protocol": { "type": "string" } + } + } + } + } + }, + "nereid": { + "type": "object", + "properties": { + "configPath": { + "type": "string" + }, + "config": { + "type": "object", + "properties": { + "server": { + "type": "object", + "properties": { + "bind_addr": { + "type": "string" + }, + "http_addr": { + "type": "string" + }, + "ml_backends_path": { + "type": "string" + } + } }, - "required": [ - "name", - "port", - "protocol", - "targetPort" - ] + "models": { + "type": "array" + } } } - }, - "required": [ - "annotations", - "labels", - "ports" - ] + } }, "readinessProbe": { "type": "object", @@ -190,16 +190,7 @@ "successThreshold": { "type": "integer" } - }, - "required": [ - "command", - "failureThreshold", - "initialDelaySeconds", - "periodSeconds", - "reset", - "successThreshold", - "timeoutSeconds" - ] + } }, "startupProbe": { "type": "object", @@ -225,41 +216,14 @@ "port": { "type": "string" } - }, - "required": [ - "path", - "port" - ] + } }, "timeoutSeconds": { "type": "integer" } - }, - "required": [ - "failureThreshold", - "httpGet", - "initialDelaySeconds", - "periodSeconds", - "reset", - "timeoutSeconds" - ] + } } - }, - "required": [ - "affinity", - "annotations", - "args", - "command", - "image", - "modelRepository", - "nodeSelector", - "readinessProbe", - "replicas", - "resources", - "service", - "startupProbe", - "tolerations" - ] + } }, "envoy": { "type": "object", @@ -291,12 +255,7 @@ "configmap_key": { "type": "string" } - }, - "required": [ - "configmap_key", - "configmap_name", - "load_from_configmap" - ] + } }, "resources": { "type": "object", @@ -310,11 +269,7 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } }, "limits": { "type": "object", @@ -325,17 +280,9 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } } - }, - "required": [ - "limits", - "requests" - ] + } }, "annotations": { "type": "object" @@ -366,19 +313,10 @@ "targetPort": { "type": "integer" } - }, - "required": [ - "name", - "port", - "targetPort" - ] + } } } - }, - "required": [ - "ports", - "type" - ] + } }, "ingress": { "type": "object", @@ -395,13 +333,7 @@ "annotations": { "type": "object" } - }, - "required": [ - "annotations", - "enabled", - "hostName", - "ingressClassName" - ] + } }, "grpc_route_timeout": { "type": "string" @@ -424,13 +356,7 @@ "fill_interval": { "type": "string" } - }, - "required": [ - "enabled", - "fill_interval", - "max_tokens", - "tokens_per_fill" - ] + } }, "prometheus_based": { "type": "object", @@ -441,17 +367,9 @@ "luaConfig": { "type": "string" } - }, - "required": [ - "enabled", - "luaConfig" - ] + } } - }, - "required": [ - "listener_level", - "prometheus_based" - ] + } }, "loadBalancerPolicy": { "type": "string" @@ -477,34 +395,9 @@ "port": { "type": "integer" } - }, - "required": [ - "audiences", - "enabled", - "jwt_issuer", - "jwt_remote_jwks_uri", - "port", - "url" - ] + } } - }, - "required": [ - "annotations", - "args", - "auth", - "enabled", - "external_config", - "grpc_route_timeout", - "image", - "ingress", - "loadBalancerPolicy", - "nodeSelector", - "rate_limiter", - "replicas", - "resources", - "service", - "tolerations" - ] + } }, "keda": { "type": "object", @@ -539,12 +432,7 @@ "stepsize": { "type": "integer" } - }, - "required": [ - "periodSeconds", - "stabilizationWindowSeconds", - "stepsize" - ] + } }, "scaleDown": { "type": "object", @@ -558,24 +446,9 @@ "stepsize": { "type": "integer" } - }, - "required": [ - "periodSeconds", - "stabilizationWindowSeconds", - "stepsize" - ] + } } - }, - "required": [ - "cooldownPeriod", - "enabled", - "maxReplicaCount", - "minReplicaCount", - "pollingInterval", - "scaleDown", - "scaleUp", - "zeroIdleReplicas" - ] + } }, "prometheus": { "type": "object", @@ -595,13 +468,7 @@ "scheme": { "type": "string" } - }, - "required": [ - "enabled", - "port", - "scheme", - "url" - ] + } }, "enabled": { "type": "boolean" @@ -627,10 +494,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "resources": { "type": "object", @@ -644,11 +508,7 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } }, "limits": { "type": "object", @@ -659,17 +519,9 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } } - }, - "required": [ - "limits", - "requests" - ] + } }, "retention": { "type": "string" @@ -683,11 +535,7 @@ "evaluation_interval": { "type": "string" } - }, - "required": [ - "evaluation_interval", - "scrape_interval" - ] + } }, "service": { "type": "object", @@ -698,11 +546,7 @@ "servicePort": { "type": "integer" } - }, - "required": [ - "enabled", - "servicePort" - ] + } }, "configMapOverrideName": { "type": "string" @@ -727,10 +571,7 @@ "hosts": { "type": "array" } - }, - "required": [ - "hosts" - ] + } } }, "ingressClassName": { @@ -739,29 +580,9 @@ "annotations": { "type": "object" } - }, - "required": [ - "annotations", - "enabled", - "hostName", - "hosts", - "ingressClassName", - "tls" - ] + } } - }, - "required": [ - "configMapOverrideName", - "extraFlags", - "global", - "ingress", - "persistentVolume", - "releaseNamespace", - "resources", - "retention", - "service", - "useExistingClusterRoleName" - ] + } }, "serviceAccounts": { "type": "object", @@ -775,16 +596,9 @@ "name": { "type": "string" } - }, - "required": [ - "create", - "name" - ] + } } - }, - "required": [ - "server" - ] + } }, "rbac": { "type": "object", @@ -792,10 +606,7 @@ "create": { "type": "boolean" } - }, - "required": [ - "create" - ] + } }, "alertmanager": { "type": "object", @@ -803,10 +614,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "pushgateway": { "type": "object", @@ -814,10 +622,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "kube-state-metrics": { "type": "object", @@ -825,10 +630,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "prometheus-node-exporter": { "type": "object", @@ -836,10 +638,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "prometheus-pushgateway": { "type": "object", @@ -847,10 +646,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "configmapReload": { "type": "object", @@ -861,30 +657,11 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } } - }, - "required": [ - "prometheus" - ] + } } - }, - "required": [ - "alertmanager", - "configmapReload", - "enabled", - "external", - "kube-state-metrics", - "prometheus-node-exporter", - "prometheus-pushgateway", - "pushgateway", - "rbac", - "server", - "serviceAccounts" - ] + } }, "grafana": { "type": "object", @@ -904,10 +681,7 @@ "enabled": { "type": "boolean" } - }, - "required": [ - "enabled" - ] + } }, "rbac": { "type": "object", @@ -915,10 +689,7 @@ "create": { "type": "boolean" } - }, - "required": [ - "create" - ] + } }, "serviceAccount": { "type": "object", @@ -926,10 +697,7 @@ "create": { "type": "boolean" } - }, - "required": [ - "create" - ] + } }, "datasources": { "type": "object", @@ -969,33 +737,14 @@ "tlsSkipVerify": { "type": "boolean" } - }, - "required": [ - "timeInterval", - "tlsSkipVerify" - ] + } } - }, - "required": [ - "access", - "isDefault", - "jsonData", - "name", - "type", - "url" - ] + } } } - }, - "required": [ - "apiVersion", - "datasources" - ] + } } - }, - "required": [ - "datasources.yaml" - ] + } }, "dashboardProviders": { "type": "object", @@ -1035,33 +784,14 @@ "path": { "type": "string" } - }, - "required": [ - "path" - ] + } } - }, - "required": [ - "disableDeletion", - "editable", - "folder", - "name", - "options", - "orgId", - "type" - ] + } } } - }, - "required": [ - "apiVersion", - "providers" - ] + } } - }, - "required": [ - "dashboardproviders.yaml" - ] + } }, "dashboardsConfigMaps": { "type": "object", @@ -1069,10 +799,7 @@ "default": { "type": "string" } - }, - "required": [ - "default" - ] + } }, "grafana.ini": { "type": "object", @@ -1083,10 +810,7 @@ "disable_login_form": { "type": "boolean" } - }, - "required": [ - "disable_login_form" - ] + } }, "auth.anonymous": { "type": "object", @@ -1097,11 +821,7 @@ "org_role": { "type": "string" } - }, - "required": [ - "enabled", - "org_role" - ] + } }, "dashboards": { "type": "object", @@ -1109,10 +829,7 @@ "default_home_dashboard_path": { "type": "string" } - }, - "required": [ - "default_home_dashboard_path" - ] + } }, "server": { "type": "object", @@ -1120,18 +837,9 @@ "root_url": { "type": "string" } - }, - "required": [ - "root_url" - ] + } } - }, - "required": [ - "auth", - "auth.anonymous", - "dashboards", - "server" - ] + } }, "resources": { "type": "object", @@ -1145,11 +853,7 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } }, "requests": { "type": "object", @@ -1160,17 +864,9 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } } - }, - "required": [ - "limits", - "requests" - ] + } }, "ingress": { "type": "object", @@ -1201,42 +897,15 @@ "hosts": { "type": "array" } - }, - "required": [ - "hosts" - ] + } } }, "annotations": { "type": "object" } - }, - "required": [ - "annotations", - "enabled", - "hostName", - "hosts", - "ingressClassName", - "path", - "pathType", - "tls" - ] + } } - }, - "required": [ - "adminPassword", - "adminUser", - "dashboardProviders", - "dashboardsConfigMaps", - "datasources", - "enabled", - "grafana.ini", - "ingress", - "persistence", - "rbac", - "resources", - "serviceAccount" - ] + } }, "metricsCollector": { "type": "object", @@ -1265,11 +934,7 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } }, "requests": { "type": "object", @@ -1280,17 +945,9 @@ "memory": { "type": "string" } - }, - "required": [ - "cpu", - "memory" - ] + } } - }, - "required": [ - "limits", - "requests" - ] + } }, "service": { "type": "object", @@ -1301,11 +958,7 @@ "type": { "type": "string" } - }, - "required": [ - "port", - "type" - ] + } }, "ingress": { "type": "object", @@ -1322,36 +975,9 @@ "annotations": { "type": "object" } - }, - "required": [ - "annotations", - "enabled", - "hostName", - "ingressClassName" - ] + } } - }, - "required": [ - "enabled", - "host", - "image", - "ingress", - "port", - "resources", - "service" - ] + } } - }, - "required": [ - "envoy", - "grafana", - "keda", - "metricsCollector", - "nameOverride", - "prometheus", - "scaleFromZero", - "serverLoadMetric", - "serverLoadThreshold", - "triton" - ] + } } \ No newline at end of file diff --git a/helm/supersonic/values.yaml b/helm/supersonic/values.yaml index 037b89ac..2ebe6963 100644 --- a/helm/supersonic/values.yaml +++ b/helm/supersonic/values.yaml @@ -10,13 +10,13 @@ serverLoadMetric: "" # -- Threshold for the metric serverLoadThreshold: 100 -# -- On RepositoryIndex, scale Triton to at least max(1, keda.minReplicaCount) replicas -# and return the index only after Envoy has a healthy Triton upstream. +# -- On RepositoryIndex, scale the inference server to at least max(1, keda.minReplicaCount) replicas +# and return the index only after Envoy has a healthy inference server upstream. # Requires keda.enabled and envoy.enabled. scaleFromZero: # -- Enable scale from zero enabled: false - # -- Seconds to wait for a healthy Triton upstream before rejecting RepositoryIndex + # -- Seconds to wait for a healthy inference server upstream before rejecting RepositoryIndex readyTimeoutSeconds: 300 # -- Seconds to keep KEDA minReplicaCount at the wake target after the last RepositoryIndex # is answered (the hold is refreshed once the server is ready, so it is measured from @@ -27,14 +27,20 @@ scaleFromZero: # -- Image for the admission sidecar on the Envoy pod admissionImage: python:3.14-slim -triton: - # -- Number of Triton server instances. Unused when scaleFromZero.enabled is true. +inferenceServer: + # -- Inference server implementation. Supported values: ``triton``, ``nereid``. + # Nereid speaks the same KServe v2 gRPC protocol as Triton, so only the image, + # launch command, model configuration and probe endpoints differ between them. + # See ``values/values-nereid.yaml`` for a complete Nereid example. + type: triton + + # -- Number of inference server instances. Unused when scaleFromZero.enabled is true. replicas: 1 - # -- Docker image for the Triton server + # -- Docker image for the inference server image: "nvcr.io/nvidia/tritonserver:26.08-py3-min" - # -- Command and arguments to run in Triton container + # -- Command and arguments to run in the inference server container command: ["/bin/sh", "-c"] args: - | @@ -43,7 +49,7 @@ triton: --log-verbose=0 \ --exit-timeout-secs=60 - # -- Resource limits and requests for each Triton instance. + # -- Resource limits and requests for each inference server instance. # You can add necessary GPU request here. resources: limits: @@ -53,16 +59,16 @@ triton: cpu: 1 memory: "2G" - # -- Annotations for Triton pods + # -- Annotations for inference server pods annotations: {} - # -- Node selector for Triton pods + # -- Node selector for inference server pods nodeSelector: {} - # -- Tolerations for Triton pods + # -- Tolerations for inference server pods tolerations: [] - # -- Affinity rules for Triton pods - another way to request GPUs + # -- Affinity rules for inference server pods - another way to request GPUs affinity: {} # -- Model repository configuration @@ -91,17 +97,44 @@ triton: # server: # path: + ## Option 5: mount models from a ConfigMap. Suitable for small models and + ## for testing; a ConfigMap is limited to ~1 MiB in total. + # storageType: "configMap" + # configMap: + # name: + service: labels: {} annotations: {} - # -- Ports for communication with Triton servers + # -- Ports for communication with inference servers ports: - { name: http, port: 8000, targetPort: 8000, protocol: TCP } - { name: grpc, port: 8001, targetPort: 8001, protocol: TCP } - { name: metrics, port: 8002, targetPort: 8002, protocol: TCP } - # -- Custom readiness probe configuration + # -- Nereid server configuration, used only when ``type`` is ``nereid``. + # Ignored for Triton, which is configured through ``command``/``args`` instead. + nereid: + # -- Path the rendered ``nereid.yaml`` is mounted at. Nereid reads this file + # from its working directory, which is ``/nereid`` in the official image. + configPath: /nereid/nereid.yaml + + # -- Contents of ``nereid.yaml``, rendered into a ConfigMap verbatim. + # ``ml_backends_path`` is resolved relative to the working directory, so use + # an absolute path when models come from ``modelRepository``. + # ``bind_addr``/``http_addr`` default to Triton's port numbers so that the + # Service, Envoy config and ServiceMonitor need no per-server special-casing. + config: + server: + bind_addr: "[::]:8001" + http_addr: "[::]:8002" + ml_backends_path: "ml-backends" + models: [] + + # -- Custom readiness probe configuration. + # The handler is whichever of ``command`` (exec shorthand), ``exec``, + # ``httpGet`` or ``tcpSocket`` is set; ``reset: true`` drops the probe. readinessProbe: # -- If true, will reset settings to k8s defaults (other readinessProbe settings will be ignored) reset: false @@ -112,7 +145,8 @@ triton: timeoutSeconds: 15 successThreshold: 1 - # -- Custom startup probe configuration + # -- Custom startup probe configuration. + # Same handler options as ``readinessProbe``. startupProbe: # -- If true, will reset settings to k8s defaults (other startupProbe settings will be ignored) reset: false @@ -223,12 +257,12 @@ keda: # -- Enable autoscaling (requires Prometheus to also be enabled). # Autoscaling will be based on the metric from parameter ``serverLoadMetric``; - # new Triton servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. + # new inference servers will spawn if the metric exceeds the threshold set by ``serverLoadThreshold``. enabled: false - # -- Minimum and maximum number of Triton servers. + # -- Minimum and maximum number of inference servers. # Set minReplicaCount to 0 to release all resources when idle (requires scaleFromZero.enabled). - # With scaleFromZero, a RepositoryIndex request scales Triton to max(1, minReplicaCount), + # With scaleFromZero, a RepositoryIndex request scales the inference server to max(1, minReplicaCount), # and upgrades keep the live ScaledObject minReplicaCount. minReplicaCount: 1 maxReplicaCount: 2 @@ -240,7 +274,7 @@ keda: cooldownPeriod: 120 # -- If set to true, KEDA sets idleReplicaCount to 0. - # This cannot scale from 0 back to 1 when the load metric comes from Triton. + # This cannot scale from 0 back to 1 when the load metric comes from the inference server. # Use scaleFromZero.enabled with keda.minReplicaCount: 0 instead. zeroIdleReplicas: false diff --git a/tests/envoy-config-test.yaml b/tests/envoy-config-test.yaml index af6f4346..457065df 100644 --- a/tests/envoy-config-test.yaml +++ b/tests/envoy-config-test.yaml @@ -42,7 +42,7 @@ data: - match: prefix: "/" route: - cluster: triton_grpc_service + cluster: inference_server_grpc_service timeout: 0s http_filters: @@ -52,7 +52,7 @@ data: http2_protocol_options: {} clusters: - - name: triton_grpc_service + - name: inference_server_grpc_service connect_timeout: 0.25s type: STRICT_DNS # Absolute FQDN (trailing dot) + V4_ONLY: matches the chart template so c-ares @@ -65,11 +65,11 @@ data: explicit_http_config: http2_protocol_options: {} load_assignment: - cluster_name: triton_grpc_service + cluster_name: inference_server_grpc_service endpoints: - lb_endpoints: - endpoint: address: socket_address: - address: supersonic-triton.cms.svc.cluster.local. + address: supersonic-inference-server.cms.svc.cluster.local. port_value: 8001 \ No newline at end of file diff --git a/tests/nereid-infer-job.yaml b/tests/nereid-infer-job.yaml new file mode 100644 index 00000000..55398189 --- /dev/null +++ b/tests/nereid-infer-job.yaml @@ -0,0 +1,71 @@ +# Drives Nereid through Envoy with a stock tritonclient. +# +# tritonclient is pip-installed from PyPI rather than taken from the Triton SDK +# image on purpose: the point is that an unmodified KServe v2 client, built from +# Triton's own proto stubs, works against Nereid unchanged. Tensor names and +# shapes come from ModelMetadata rather than being hardcoded, so the job tests +# the metadata path too. +# +# perf_analyzer is deliberately not used here: it needs the model-config and +# statistics RPCs, which Nereid does not implement. +apiVersion: batch/v1 +kind: Job +metadata: + name: nereid-infer-job + namespace: cms +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: infer + image: python:3.11-slim + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + pip install --quiet --no-cache-dir "tritonclient[grpc]" numpy + python3 - <<'PY' + import numpy as np + import tritonclient.grpc as grpcclient + + URL = "supersonic.cms.svc.cluster.local:8001" + MODEL = "model3" + + client = grpcclient.InferenceServerClient(URL) + + # Envoy may still be settling on its upstream just after rollout. + import time + for attempt in range(30): + try: + meta = client.get_model_metadata(MODEL) + break + except Exception as exc: + print(f"metadata attempt {attempt}: {exc}", flush=True) + time.sleep(2) + else: + raise SystemExit("FAIL: model metadata never became available") + + print("server metadata:", client.get_server_metadata(), flush=True) + print("model metadata:", meta, flush=True) + + inp_meta, out_meta = meta.inputs[0], meta.outputs[0] + # -1 marks a client-provided dimension; make it concrete. + shape = [d if d > 0 else 1 for d in inp_meta.shape] + count = int(np.prod(shape)) + values = np.arange(1, count + 1, dtype=np.float32).reshape(shape) + + tensor = grpcclient.InferInput(inp_meta.name, shape, inp_meta.datatype) + tensor.set_data_from_numpy(values) + result = client.infer(model_name=MODEL, inputs=[tensor], request_id="ci-1") + + out = result.as_numpy(out_meta.name) + if out is None or out.size == 0: + raise SystemExit(f"FAIL: empty output for {out_meta.name!r}") + print(f"OK: in {shape} {inp_meta.datatype} -> out {list(out.shape)} {out.dtype}", flush=True) + PY + resources: + requests: + cpu: 200m + memory: 256Mi diff --git a/tests/nereid-model-fixture.yaml b/tests/nereid-model-fixture.yaml new file mode 100644 index 00000000..21e1137e --- /dev/null +++ b/tests/nereid-model-fixture.yaml @@ -0,0 +1,22 @@ +# Model fixture for the Nereid CI job: NVIDIA-free, self-contained. +# +# This is nereid-server's own `ml-backends/model3` example (a TorchScript MLP +# taking a [16] FP32 tensor), vendored so CI needs no model volume, no CVMFS +# and no network fetch. It is served by Nereid's Rust `torch` backend, which is +# in the image's default feature set and needs no writable model directory -- +# unlike the Python backend, which builds a venv/ inside the model folder. +# +# Upstream: https://github.com/ngpaladi/nereid-server/tree/main/ml-backends/model3 +apiVersion: v1 +kind: ConfigMap +metadata: + name: nereid-model-fixture + namespace: cms +data: + model_inference.textproto: | + # Per-model inference config (example) + # Message type: inference.ModelInferenceConfig + input_shape: [16] + max_batch_size: 10 +binaryData: + mlp.pt: UEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAAKABgAbWxwL2RhdGEvMEZCFABaWlpaWlpaWlpaWlpaWlpaWlpaWixDkz1ODB4+CNhzveA/v72AA289XG6mvaQoKD5A7PA75JfuvWAgDT0M+wm+VOEBvugPUD3uWDq+GNfhPaBEkT2Yk1K9oOUuPIKHNz5oEIU94LRTPKh/CD3M5GK+IEsmvpBkTb1UYZ+9Pi5uvvizCj5WvTs+gLfHu8TxDL6AGx8+GFTiPWQeAb5o6qg9aihRvggh8b2GNFm+WE+hvUR0+j387uK9FJ0dvkDX+rz4ODQ+JOa4PTCh+r12XD8+LILzvYBNDj1+UBO+yO0qvrDvrL24SNy9YFwovCRcSL6YBWI+/Ea4PUhhMr7A4Hu9oGNWvAb5VL5mjEE+OGtNvSZPFD6sw2A+LOMyvmY3Bz6Q9ha+1CXfPfhsML0Ox3E+ZnkHPsx/wL1qjjm+mGaQvfY0ZD6sUVo+pNVpPgAQBzm8pto98C46Pt5UKb7M5m++IIFePfAgFb747209hMToPRhuSD1gdF28UMACveBafT4aQhK+WEimPd5eCr6mJnk+RJvFvYp1YT5QCiI+ALt7vFDopb269lG+/h9dvoJWCL72hWW+xIDcvSwpiL14iLG9jrB2vmDqfj3Qiec9qqtRvrQEqT3Ipvk9UMPGvaKJI76gakk+LiYUvqA7Mj04DZ+9AKAROwQwX76aezu+/JzBvVTIQz7QujG9ICZzvBzbf77IwQe9SkRXPkCAbDxWix++YDgaPpDAPD3uR24+lNgNviQ5p73KhiK+bHZ5vlSPjz3k4mK+bFuZvSBTTT7maTm+zLI2vvAyez1a8BQ+QH1TveqMMb4omJW9KqdNvmAW27ywLLy94B7KPNjDnj0ccuE90gwOvuCph70Q7OO94CB2PD4fEr42QAI+gDfaPSSBCz6I4OE9mIZWvkAXhbuS5GI+/PjAvUxErL3w6Ju8hC46vkJySr7+pRG+yt9zvqRYhD1+5VS+jDCOvdCxUT4Mk3O+YB1VPAClPz4A33c9ZPW5PUKQbj5gTxC8QDS9PdKDAj5Ghy0+AJhUuYwRrD1gOjA9sIyYPIYDbz5IqQ4+8DfqvTS/XD5CQDO+4LJ+vPCpkz06Szc+zGdmPvpwID4ghCw+sMUHPZCsNT3kxay90Ke7PQAMArqUd3S+WMWZvfb4Fj5SAVo+LCPXPTZQQD76fQY+VCR4PvxlkD3aCm++/BSzPS5GND4smM69qPARvoBZkjtANzI+KAiYPTBm9T2o7lw9Qp8rPsZWED5QOT89gCpKvZDgxb2g6iM8AMGSPIDzRj7AUOe7IA5XPdRlJr6OjSw+YHrWvGCCwL1MpI29ruo8voCgBztoKjg+8OXFvARrlD0cnGy+9IWvPeg8Kj10irK92BzQvWCKCz4g9hS8uKLCPTDdUL4stVa+uFkJPih0q73Gxhg+cIb+PaL9SD6slyY+0LHdPXwv8b2Qm4w8zAM5PthY973aKQ8+iPJJPXYpPj6gJ9O9KIAYPYDH970AiYs84MNTvFgtZj7Y8D6+FAJUvgCxUb5Ek+u9/FgXvmSZzL1wv2I+0CxzvcLyID6MgEW+BF5kPhwf2D3Y0k++AC4JOpj9UD7A17C8ULpoPmpbcT7ogg69vK9+vlDau72I4t69iBbgPcAa7L1CWBG+etkZPnpoID6wM1q+MCy4PYbvTj5u4H8+UOpBvSB82jw0tZG9codwPk5DYL6S1Fe+kjh3vpxXBj5Iqc69wInOOzqrED4AuuY77NZMPsyzEj7g+cG8DPLOPdxj9D3cMx0+hn1VPogVVD6AE/K7aBpQPnhPC76YqKS9WLUGPrhKxT1wQmG+1pFxPsDNBbzw5E8+gL9RPrgxQz16Ene+fCEcvsA1U7xEqQe+7ECXPdSaFT6oKPu92NNqveocBD5gepM8QLCePFBBIT0Ifr69GHYFPvh18b1w6OE88EJGvswSML6InpI9vkQ8PsTkuz1KHzA+mtY5PgASHr28YLo9sBrBvQw5Ej4GSBK+WAQEPhrSb77uDwm+AO3QurSoUz4kYGQ+6GbrvZqBGb5gol2+lgZhPtgWN72IetK9/HpaPuRM6D0EsZs9wHPEvAZ4XD4E8JQ93PXAPSQSeD5oRE6+OBFRveSxY74EFiw+qCZuPZL2KD4o5i6+nhNfPhLwPz5EiGu+ZNkAPoD88juAcyo+0BYovlBu2DxyvGC++Gd1PvwwOL4ob5u9nObdPXqSYj5gf6i9WDJ9vdyGqr1YOx29VjcZPqDB/zy+4Eg+bNHAvQBqHDrcVVW+sHfivYr7bD7gU/u8oK6evBzajL0ARXM86LZ5vngypr10NzU+yLhHPly6gr146fE9tOQDPtTlTj4AHKm7KLGxvVBmPj6sm36+5s4rvnBF/b1CIWM+vntYPlxR1r0YDko91lhlPoAqF7w0mPI9CH7KvfAYLr043B896BlZvcDGWjwQg1K9gACqvJaGdL7AevC8sEahvchdOD5AbKW9Vi8bvtxRMD60nk4+4kg9PvQqM76cC5O9tJnRvbQ7wz2QP3u+CJXKPbhlML14j5e9cHM2vWA+OD2w4Uu9EgdevjJKQ75k82g+Gjw5PlCWEL7g3F68JJqPvX5vUr42qk4+APEjvOA9Y72u122+/rMkPkidRT3kvIE97EANPmATAj4gVwo9YCh1vKDjfz0K3GS+XCnXvRQfUT4U4ZY94Pn+PGJ+Bb7kzPm9zPztvYDXqr3wAeY96mIIvmopSr4IEmK9dDEXPop9Ub4gmOG9sPabPaw0kj2goC+8UEsHCPwU+YwACAAAAAgAAFBLAwQAAAgIAAAAAAAAAAAAAAAAAAAAAAAACgAIAG1scC9kYXRhLzFGQgQAWlpaWmBLPb3ogCG9+HKfPeyX5r1cJ34+EM7XPT5nH74AzLS88PFhvrS49r36lm0+7NFNPlCsPj1AbPQ8YBrwvHTktz1UuUk+ULqNvNArj73GYnQ+8OPLvKBvIzwkNY89FOGpPSgRF77SlQ8+Khl9vqA8lTxIpjq9SDFcPgRJ+j2YmAq9UEsHCCZQP2GAAAAAgAAAAFBLAwQAAAgIAAAAAAAAAAAAAAAAAAAAAAAACgAIAG1scC9kYXRhLzJGQgQAWlpaWvaV/7xaoeo91LkCvs8u6ryRhTe9/DG9vdYG0bwAxYY9mUteu8cTJj5GqcS98bQHvvQ8N72aQGW9ODmWvWBgorwo3gk+CvWdPRz6xT0atKa9toffPfpWMT5fMg++IdNYvTTQsDsV5Ji8MsvUvI3PGT0XYv49bqUmPk+G6T17qvq9EWbbvaOqHr7kKyM+7KMmvkR7Gj3WOZa98WqqvP8pFb1KaNi9e8waPpjEA75cu6I91XW5veU4EL2yR1C9VBTEvfdb1b1/hL28ly2wvcMMFb7vMwC+bpuePfW9zL2v3zY8sTTjPcEEb7213309ohtyPQXaar0WX5c8r7q6PS19F75eeAm+g4VWvTrw073u9FG9epf4vKhnzrxIaLq9jm+BPES5GL5kGlG9ueQ2vaQLlL2dEBM+4VBSPWbh0b0yf5+9DcEoPq06Mr4X4y09pnowPjDJnz3Z2++9xegHPpPqsjz7Br49o7SZvSMQEb6li2G9qYeSvJ3oCb5DzUU9utggPnL7Uj02Ba081soTPglLGz7pequ9VU8KvsRaLb7Ny8S9wvYPvYawzL2xxs29nBYsPm4E8r18Nke90Pe4PWtSgT1n0Ro+JdUaPqF6Dr4qTCa+t0otPm50cD1Qb6295aa8vVKqp72A5w29s8YrPtl5c70JAxk+vJ7MPXeOtLzKjJQ9UEsHCMm2pK0AAgAAAAIAAFBLAwQAAAgIAAAAAAAAAAAAAAAAAAAAAAAACgAIAG1scC9kYXRhLzNGQgQAWlpaWnZFLr5QG9S940gTPkFa8r1QSwcIiht2rBAAAAAQAAAAUEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAAMADYAbWxwL2RhdGEucGtsRkIyAFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpagAJjX190b3JjaF9fCk1MUApxACmBfShYCAAAAHRyYWluaW5ncQGJWBYAAABfaXNfZnVsbF9iYWNrd2FyZF9ob29rcQJOWAMAAABuZXRxA2NfX3RvcmNoX18udG9yY2gubm4ubW9kdWxlcy5jb250YWluZXIKU2VxdWVudGlhbApxBCmBfShoAYloAk5YAQAAADBxBWNfX3RvcmNoX18udG9yY2gubm4ubW9kdWxlcy5saW5lYXIKTGluZWFyCnEGKYF9KFgGAAAAd2VpZ2h0cQdjdG9yY2guX3V0aWxzCl9yZWJ1aWxkX3RlbnNvcl92MgpxCCgoWAcAAABzdG9yYWdlcQljdG9yY2gKRmxvYXRTdG9yYWdlCnEKaAVYAwAAAGNwdXELTQACdFFxDEsAKEsgSxB0KEsQSwF0iGNjb2xsZWN0aW9ucwpPcmRlcmVkRGljdApxDSlSdFJxDlgEAAAAYmlhc3EPaAgoKGgJaApYAQAAADFxEGgLSyB0UXERSwAoSyB0KEsBdIhoDSlSdFJxEmgBiWgCTnVicRNoEGNfX3RvcmNoX18udG9yY2gubm4ubW9kdWxlcy5hY3RpdmF0aW9uClJlTFUKcRQpgX0oaAGJaAJOdWJxFVgBAAAAMnEWY19fdG9yY2hfXy50b3JjaC5ubi5tb2R1bGVzLmxpbmVhci5fX190b3JjaF9tYW5nbGVfMApMaW5lYXIKcRcpgX0oaAdoCCgoaAloCmgWaAtLgHRRcRhLAChLBEsgdChLIEsBdIhoDSlSdFJxGWgPaAgoKGgJaApYAQAAADNxGmgLSwR0UXEbSwAoSwR0KEsBdIhoDSlSdFJxHGgBiWgCTnVicR11YnEedWJxHy5QSwcIxxRtgZACAACQAgAAUEsDBBQACAgIAAAAAAAAAAAAAAAAAAAAAAAVAC0AbWxwL2NvZGUvX190b3JjaF9fLnB5RkIpAFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaVU9NqgIxDN57iixnwNcDDPhO8OQJuhMJnZlUizXRtEWPb1sUdJXky/eTTMHGCOu/TbeWOQfqhwUA4tWqvVAijYiwgv2hoWN27gtKaj17PsIAo0ioJB/R5RBwtNP5bnXGk8i57P+vyQvbsK/EqmVKBUZMotMJ0bRqmM2lHRLNJJyKPanZ0i0TJ29rwkwOnGj17iIF9+lR/lgWCsBjgB1xFO3h5/fVDm1TY1dQhaa0DVJKWRm6Mvfmbf1YQr94AlBLBwhXkpL2vgAAAB4BAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAAB8ANQBtbHAvY29kZS9fX3RvcmNoX18ucHkuZGVidWdfcGtsRkIxAFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlqAAlgYAAAARk9STUFUX1dJVEhfU1RSSU5HX1RBQkxFcQBYAAAAAHEBhXECSwBjdG9yY2guaml0Ll9waWNrbGUKYnVpbGRfaW50bGlzdApxAyhdKGV0UksASwCHcQRLAEsAh0sAh4VxBYcuUEsHCLHxvT1qAAAAagAAAFBLAwQUAAgICAAAAAAAAAAAAAAAAAAAAAAAMAA4AG1scC9jb2RlL19fdG9yY2hfXy90b3JjaC9ubi9tb2R1bGVzL2NvbnRhaW5lci5weUZCNABaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpajZFNasMwEIX3OcWQlQ2psNOdoT1BSqE/qxCGiTNORORRKsltjl9ZrnFLQlsQSDPv6dOTVBvyHp75rWMJmkz2YHed4byaASCeyFHLgZ1HhDtYb1J32zXNj1ZwpEXLHirYWmt6k/bYdMbglurjB7kdHqw9Rv3xFLQVMuveOOBIxAbq2xG5nhfzTeQiBuvqA6JKsxJRbUrmldHC5NQqTdcI5V8EqoN+T371xKvXa4zlP1PgaGlJ9oaxmHLtuIHGuv72mWfTVL/haishPiI7Nf3FIkIAzhW8sHjrcri5/1pWScFljLjnmDq4dMICYu580MpLrRy14lIrRu02ahkuczVGz7D8XhRTcV5APoy01XHonETC7BNQSwcIHBiuWBMBAABVAgAAUEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAA6AAUAbWxwL2NvZGUvX190b3JjaF9fL3RvcmNoL25uL21vZHVsZXMvY29udGFpbmVyLnB5LmRlYnVnX3BrbEZCAQBagAJYGAAAAEZPUk1BVF9XSVRIX1NUUklOR19UQUJMRXEAWAAAAABxAYVxAksAY3RvcmNoLmppdC5fcGlja2xlCmJ1aWxkX2ludGxpc3QKcQMoXShldFJLAEsAh3EESwBLAIdLAIeFcQWHLlBLBwix8b09agAAAGoAAABQSwMEFAAICAgAAAAAAAAAAAAAAAAAAAAAAC0AOwBtbHAvY29kZS9fX3RvcmNoX18vdG9yY2gvbm4vbW9kdWxlcy9saW5lYXIucHlGQjcAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWk2PQY7CMAxF95zC6qqVOjlAJeYEM2LDDqEobR0aEWLkpILjkzgFsYr9/fLtP3kTI/y5gIbbf5pXj92wA9D6btjcMCFHrWEPp+aB7rKkpodmdCbm9yzcuFr7gYpUORjgiCESZ6Xw331i44ILl6yNRL7YuKjt6r0ezXR9GJ71QnTN88M9OQrGnwpY3Ge0YIkL00b0dsgnJOJp0VrJq0JQNwkSlZdcqsbr82+A5/uODn5+t3KQiVy5h2KqSi3iFmaTaycDxrRygLqy7mmf/cb3YtbtXlBLBwhb57gi0gAAAF4BAABQSwMEFAAICAgAAAAAAAAAAAAAAAAAAAAAADcACQBtbHAvY29kZS9fX3RvcmNoX18vdG9yY2gvbm4vbW9kdWxlcy9saW5lYXIucHkuZGVidWdfcGtsRkIFAFpaWlpazZRLj9MwFIWdzrMpzx1LlmXRuGmhBYQQw2NgZMqjU8CbkeUmZuqOG9ux22p2LBBZsEEggcSGv4qTlNGwB6WSpSPL9r2fj4/8oYavAQD2Xw4HeyPy7mD0jByOhgcvnpLR3sPnTzRoYrcMtIe/OYFvDEsNjPmCTwk9pumSCvhYLhMhaWzg28NHMmb7XDAD1amdyISoVE5ZZA189XpOEwuVzqXFkgUUfLza1Q3CDjTcspai0Qk9dsetTKMJTBI4k/E8ryd4wmgaqNNm2L154+7199I1T2Nf1/DPaslKLcj6vZ5DI0bIJTkD3MA/1gfwdj8HjKgQhM+U8PUm/rU+dP3CvmVKlWLxecot/L1aykgmlroIFgns3OqeT+A2Hv8TOJPflbVmQrUMXZSW/BX1Hfy1Ahem3EJiUxqVRJ2w7ZiKOSnd8fUu/lI5WL8d5uEpwcrU1PHn6v1qh70/fvnax9F/i8od1+Ze+SL3fd3AH9fkw9YXrL6IQFTsCJw7AVE8OnG5Gc+5iAlPrODG+vpS86jJ7BABBDJ9uRA3Bg+8Sb6CPFRDG2gTba10e6U7aBfVkY9AfraBPmX6CgKDRj1DXjY48iZnlTJ9NQt+A1BLBwh2sEIxyQEAAPEGAABQSwMEFAAICAgAAAAAAAAAAAAAAAAAAAAAADEAGABtbHAvY29kZS9fX3RvcmNoX18vdG9yY2gvbm4vbW9kdWxlcy9hY3RpdmF0aW9uLnB5RkIUAFpaWlpaWlpaWlpaWlpaWlpaWlpaVY7BygIxDITvPkWOu/D/Ba8FfQJFED2JhOxuqsVuI2mrr+92FcRTwuSbyfSBUoI9b47NVoYSuLULAMQ7KY2cWRMirOB0ntWuOPcjZSUffbyAhU4kVMgndCUE7Ki/PUkHvIrcpvvunr1ECqcKVu/ADpxoZZrEwdnpQRbtr4hmniZGM86lkqE++wfVBFPL/k1+ANJLGTlmXFo4cEyiLfyvP6udEeVcNMI7TzmU5mtqFy9QSwcIHeYhkbAAAAD+AAAAUEsDBBQACAgIAAAAAAAAAAAAAAAAAAAAAAA7ACcAbWxwL2NvZGUvX190b3JjaF9fL3RvcmNoL25uL21vZHVsZXMvYWN0aXZhdGlvbi5weS5kZWJ1Z19wa2xGQiMAWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlrNlU1v0zAYx7Nu69ZsvJ44ciyH1k03WkAIabwMJlNeugK+IMtNvNWda8ex02q33XJASIgDSAiJz8En4lMgYSdl6u6bMinSX9YT27/88sg+qaBbnuftvu73dgb4w97gBd4f9PdePceDnccvnymvjmzZU0vosw3wTtNEg4hN2RiTQ5LMCAdP5UxwSSIN3u8/kRHdZZxqEB+bkRQ4TuSYhkaDN29TIgyIlYsGFVPA2XD+1lYzaAPNDG3EJDwih3a6kUk4AkKAg1SEhklBeDM+rgfddnDnwe2E8tRXFfS9HKaJjFK3HrFkU+LocrbtLYt2IC1AEvlqGf0ol67IwlqnY9Gw5nKGTwFXyta3CHiv6wBDwjlmk5j7ahX9ujx03VzfLCFxTKNFyir6Vi5lKIUhTNDEgbbvnunANTQ8FzjtvpU2JjxuaDItlGwvbrSOvpZgYcwMwCYhYUHUDlqWKR/jwo6vauhL6WDdljuxivG8a3z0qXxfraDz35evNlB4Ya1y327zsPgjj3y1iU4uwUWirhh1FXphXm1aM00cs/DI9swwZTzCTBjOtPHVtfrHOjV96EEvU9fzcM+fkSvAJViBy3AFrsLqPNfmuQ5r0Icb0HOTN3u/q5m6Ab3ez1oGKxn8OzpdKlM3s+Y/UEsHCER9aFfnAQAAhgcAAFBLAwQUAAgICAAAAAAAAAAAAAAAAAAAAAAAPwAsAG1scC9jb2RlL19fdG9yY2hfXy90b3JjaC9ubi9tb2R1bGVzL2xpbmVhci9fX190b3JjaF9tYW5nbGVfMC5weUZCKABaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaTY+7bsMwDEX3fAXhyQZcIV0NpF/Qoku3oCBom7KFyFJAycjvVw+n7UTy8vBxJ0shwLtxTNJ++Hm33A0nAMQ7CW0cWQIiXODaPNgsa2x6aEZDIcXvwo271r9QlioHA3yxC16Skvn/dRQyzrglaaP3Nq8xAfVuLY403R4kM67e31L/8x6Nd2SvGczbZ9agvWSmDWz1kF6IXqYVUZWonFNbMRKULb4UPpGN3GIZz6oa7tM+AJJl39hFfH2+2MHL25EOBSkGLpDvqZwX8fB5yLUqDTwnsf5SH2j/TvTHWF92doUXjru4NHb6AVBLBwiAPG746AAAAI4BAABQSwMEFAAICAgAAAAAAAAAAAAAAAAAAAAAAEkAIQBtbHAvY29kZS9fX3RvcmNoX18vdG9yY2gvbm4vbW9kdWxlcy9saW5lYXIvX19fdG9yY2hfbWFuZ2xlXzAucHkuZGVidWdfcGtsRkIdAFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpazZTNb9MwGMbddl9N+bxx5FgOjZsWWkAIaXwMJlMYXQFfkOUmZnXnxnbsttptB0QOXBBIIHHhX8VJyjTum1LJ0iPL9vv+/PiRT6v4FgBg781wsDsiH/ZHL8nhaLj/+gUZ7T559VyDJnbLQFfwDyfwnWGJgRFf8CmhRzRZUgGfyWUsJI0MfH/4VEZsjwtmoDqxExkTlcgpC62BB2/nNLZQ6UxaLF5AwcerXV0/6EDDLWspGh7TI3fcyiScwDiGMxnNs3qCx4wmvjppBt27dx7e/iRd8yTydBX/Lpes0Jys3+s5NGKEXJIzwBr+tT6A9/sZYEiFIHymhKc38J/1oevn9i0TqhSLzlNu4p/lUoYyttRFME9g5173fAK38PhC4Ex2V9aaCdUydFFY8l/Ut/H3ElyYcguJTWhYEHWCtmPK56Rwx9M7+FvpYP12kIWnACtSU8dfy/erHfT++eVpD4eXFpUHrs2j4kUee7qBP6/Jh62vWH0VgTDf4Tt3fKJ4eOxyM55zEREeW8GN9fS15scms0MEEEj19VzcGBxUJtkKqqAqqqENtLnSrZVuox1URx4C2dkG+pLqGwgMGvUU1dLBaWVyVinVN1P/L1BLBwiQBGHDyQEAAPEGAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAABEAOABtbHAvY29uc3RhbnRzLnBrbEZCNABaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpagAIpLlBLBwhtLwlXBAAAAAQAAABQSwMEAAAACAAAAAAAAAAAAAAAAAAAAAAAABMAOwBtbHAvdHJhY2VkX2lucHV0cy8wRkI3AFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAABUADQBtbHAvdHJhY2VkX2lucHV0cy5wa2xGQgkAWlpaWlpaWlpagAJjdG9yY2guaml0Ll9waWNrbGUKcmVzdG9yZV90eXBlX3RhZwpxAH0oWAcAAABmb3J3YXJkcQFoAF0oY3RvcmNoLl91dGlscwpfcmVidWlsZF90ZW5zb3JfdjIKcQIoKFgHAAAAc3RvcmFnZXEDY3RvcmNoCkZsb2F0U3RvcmFnZQpxBFgBAAAAMHEFWAMAAABjcHVxBksQdFFxB0sAKEsBSxB0KEsQSwF0iWNjb2xsZWN0aW9ucwpPcmRlcmVkRGljdApxCClSdFJxCWVYCQAAAExpc3RbQW55XXEKhlJ1WBQAAABEaWN0W3N0ciwgTGlzdFtBbnldXXELhlJxDC5QSwcINbE6SQEBAAABAQAAUEsDBAAACAgAAAAAAAAAAAAAAAAAAAAAAAALAAYAbWxwL3ZlcnNpb25GQgIAWlozClBLBwjRnmdVAgAAAAIAAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAAA0AQwBtbHAvYnl0ZW9yZGVyRkI/AFpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWmxpdHRsZVBLBwiFPeMZBgAAAAYAAABQSwMEAAAICAAAAAAAAAAAAAAAAAAAAAAAABoAMgBtbHAvLmRhdGEvc2VyaWFsaXphdGlvbl9pZEZCLgBaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaWlpaMDQ4Nzg4MDEyNjc4NDI5MTgzMzkxMzgyMzE5NTA1NTkwNjk0OTEyMVBLBwjArOIDKAAAACgAAABQSwECAAAAAAgIAAAAAAAA/BT5jAAIAAAACAAACgAAAAAAAAAAAAAAAAAAAAAAbWxwL2RhdGEvMFBLAQIAAAAACAgAAAAAAAAmUD9hgAAAAIAAAAAKAAAAAAAAAAAAAAAAAFAIAABtbHAvZGF0YS8xUEsBAgAAAAAICAAAAAAAAMm2pK0AAgAAAAIAAAoAAAAAAAAAAAAAAAAAEAkAAG1scC9kYXRhLzJQSwECAAAAAAgIAAAAAAAAiht2rBAAAAAQAAAACgAAAAAAAAAAAAAAAABQCwAAbWxwL2RhdGEvM1BLAQIAAAAACAgAAAAAAADHFG2BkAIAAJACAAAMAAAAAAAAAAAAAAAAAKALAABtbHAvZGF0YS5wa2xQSwECAAAUAAgICAAAAAAAV5KS9r4AAAAeAQAAFQAAAAAAAAAAAAAAAACgDgAAbWxwL2NvZGUvX190b3JjaF9fLnB5UEsBAgAAAAAICAAAAAAAALHxvT1qAAAAagAAAB8AAAAAAAAAAAAAAAAAzg8AAG1scC9jb2RlL19fdG9yY2hfXy5weS5kZWJ1Z19wa2xQSwECAAAUAAgICAAAAAAAHBiuWBMBAABVAgAAMAAAAAAAAAAAAAAAAAC6EAAAbWxwL2NvZGUvX190b3JjaF9fL3RvcmNoL25uL21vZHVsZXMvY29udGFpbmVyLnB5UEsBAgAAAAAICAAAAAAAALHxvT1qAAAAagAAADoAAAAAAAAAAAAAAAAAYxIAAG1scC9jb2RlL19fdG9yY2hfXy90b3JjaC9ubi9tb2R1bGVzL2NvbnRhaW5lci5weS5kZWJ1Z19wa2xQSwECAAAUAAgICAAAAAAAW+e4ItIAAABeAQAALQAAAAAAAAAAAAAAAAA6EwAAbWxwL2NvZGUvX190b3JjaF9fL3RvcmNoL25uL21vZHVsZXMvbGluZWFyLnB5UEsBAgAAFAAICAgAAAAAAHawQjHJAQAA8QYAADcAAAAAAAAAAAAAAAAAohQAAG1scC9jb2RlL19fdG9yY2hfXy90b3JjaC9ubi9tb2R1bGVzL2xpbmVhci5weS5kZWJ1Z19wa2xQSwECAAAUAAgICAAAAAAAHeYhkbAAAAD+AAAAMQAAAAAAAAAAAAAAAADZFgAAbWxwL2NvZGUvX190b3JjaF9fL3RvcmNoL25uL21vZHVsZXMvYWN0aXZhdGlvbi5weVBLAQIAABQACAgIAAAAAABEfWhX5wEAAIYHAAA7AAAAAAAAAAAAAAAAAAAYAABtbHAvY29kZS9fX3RvcmNoX18vdG9yY2gvbm4vbW9kdWxlcy9hY3RpdmF0aW9uLnB5LmRlYnVnX3BrbFBLAQIAABQACAgIAAAAAACAPG746AAAAI4BAAA/AAAAAAAAAAAAAAAAAHcaAABtbHAvY29kZS9fX3RvcmNoX18vdG9yY2gvbm4vbW9kdWxlcy9saW5lYXIvX19fdG9yY2hfbWFuZ2xlXzAucHlQSwECAAAUAAgICAAAAAAAkARhw8kBAADxBgAASQAAAAAAAAAAAAAAAAD4GwAAbWxwL2NvZGUvX190b3JjaF9fL3RvcmNoL25uL21vZHVsZXMvbGluZWFyL19fX3RvcmNoX21hbmdsZV8wLnB5LmRlYnVnX3BrbFBLAQIAAAAACAgAAAAAAABtLwlXBAAAAAQAAAARAAAAAAAAAAAAAAAAAFkeAABtbHAvY29uc3RhbnRzLnBrbFBLAQIAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAATAAAAAAAAAAAAAAAAANQeAABtbHAvdHJhY2VkX2lucHV0cy8wUEsBAgAAAAAICAAAAAAAADWxOkkBAQAAAQEAABUAAAAAAAAAAAAAAAAAQB8AAG1scC90cmFjZWRfaW5wdXRzLnBrbFBLAQIAAAAACAgAAAAAAADRnmdVAgAAAAIAAAALAAAAAAAAAAAAAAAAAJEgAABtbHAvdmVyc2lvblBLAQIAAAAACAgAAAAAAACFPeMZBgAAAAYAAAANAAAAAAAAAAAAAAAAANIgAABtbHAvYnl0ZW9yZGVyUEsBAgAAAAAICAAAAAAAAMCs4gMoAAAAKAAAABoAAAAAAAAAAAAAAAAAViEAAG1scC8uZGF0YS9zZXJpYWxpemF0aW9uX2lkUEsGBiwAAAAAAAAAHgMtAAAAAAAAAAAAFQAAAAAAAAAVAAAAAAAAAFsGAAAAAAAA+CEAAAAAAABQSwYHAAAAAFMoAAAAAAAAAQAAAFBLBQYAAAAAFQAVAFsGAAD4IQAAAAA= diff --git a/tests/values-external-envoy-config.yaml b/tests/values-external-envoy-config.yaml index a83fc671..f9e57c55 100644 --- a/tests/values-external-envoy-config.yaml +++ b/tests/values-external-envoy-config.yaml @@ -1,6 +1,6 @@ # Test values for external Envoy configuration testing -triton: +inferenceServer: replicas: 1 image: fastml/triton-torchgeo:26.04-py3-geometric # works for CMSSW run3 command: ["/bin/sh", "-c"] diff --git a/tests/values-nereid-ci.yaml b/tests/values-nereid-ci.yaml new file mode 100644 index 00000000..c1928d00 --- /dev/null +++ b/tests/values-nereid-ci.yaml @@ -0,0 +1,85 @@ +# CI values for the Nereid job: a single CPU server with the vendored model3 +# fixture, behind Envoy. No CVMFS, no Prometheus, no KEDA, so the job needs +# only a bare kind cluster. + +inferenceServer: + type: nereid + + # TODO: pin a released tag once nereid-server publishes one. + image: ghcr.io/ngpaladi/nereid-server:latest + + # Nereid reads nereid.yaml; use the image's entrypoint. + command: null + args: null + + replicas: 1 + + resources: + limits: { cpu: 1, memory: 2G } + requests: { cpu: 1, memory: "512Mi" } + + service: + ports: + - { name: grpc, port: 8001, targetPort: 8001, protocol: TCP } + - { name: metrics, port: 8002, targetPort: 8002, protocol: TCP } + + nereid: + config: + server: + bind_addr: "[::]:8001" + http_addr: "[::]:8002" + ml_backends_path: "/models" + models: + - name: model3 + device: cpu + queue_capacity: 16 + + # The fixture ConfigMap is the model *folder*, so it mounts at + # / rather than at ml_backends_path itself. + modelRepository: + enabled: true + storageType: "configMap" + mountPath: "/models/model3" + configMap: + name: nereid-model-fixture + + readinessProbe: + reset: false + command: null + httpGet: + path: /v2/health/ready + port: metrics + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 10 + timeoutSeconds: 5 + successThreshold: 1 + + startupProbe: + reset: false + httpGet: + path: /v2/health/live + port: metrics + initialDelaySeconds: 0 + periodSeconds: 5 + failureThreshold: 30 + timeoutSeconds: 5 + +envoy: + enabled: true + resources: + requests: + cpu: 0.1 + memory: "128Mi" + +prometheus: + enabled: false + +grafana: + enabled: false + +keda: + enabled: false + +metricsCollector: + enabled: false diff --git a/values/values-anvil-cms.yaml b/values/values-anvil-cms.yaml index 6b696cad..e7e300ac 100644 --- a/values/values-anvil-cms.yaml +++ b/values/values-anvil-cms.yaml @@ -1,4 +1,4 @@ -triton: +inferenceServer: image: nvcr.io/nvidia/tritonserver:26.08-py3 command: ["/bin/sh", "-c"] args: diff --git a/values/values-geddes-cms.yaml b/values/values-geddes-cms.yaml index f113b11e..cec9762b 100644 --- a/values/values-geddes-cms.yaml +++ b/values/values-geddes-cms.yaml @@ -1,5 +1,5 @@ # Geddes CMS SuperSONIC. -# Triton minReplicaCount is 0. RepositoryIndex scales the first replica +# Inference server minReplicaCount is 0. RepositoryIndex scales the first replica # and waits for a healthy Envoy upstream. # # Based on Purdue AF production values: @@ -12,7 +12,7 @@ scaleFromZero: readyTimeoutSeconds: 300 holdMinReplicasSeconds: 300 -triton: +inferenceServer: image: fastml/triton-torchgeo:26.04-py3-geometric command: ["/bin/sh", "-c"] args: diff --git a/values/values-minimal-full.yaml b/values/values-minimal-full.yaml index fd65e0a9..3126503b 100644 --- a/values/values-minimal-full.yaml +++ b/values/values-minimal-full.yaml @@ -1,4 +1,4 @@ -# Full CI values. Triton starts at 0 replicas; RepositoryIndex scales the first pod. +# Full CI values. The inference server starts at 0 replicas; RepositoryIndex scales the first pod. scaleFromZero: enabled: true readyTimeoutSeconds: 300 @@ -6,7 +6,7 @@ scaleFromZero: # back to zero while the RepositoryIndex hold is still waiting for it. holdMinReplicasSeconds: 300 -triton: +inferenceServer: replicas: 1 image: fastml/triton-torchgeo:26.04-py3-geometric # works for CMSSW run3 command: ["/bin/sh", "-c"] diff --git a/values/values-minimal.yaml b/values/values-minimal.yaml index ef7c81b5..444d3899 100644 --- a/values/values-minimal.yaml +++ b/values/values-minimal.yaml @@ -1,4 +1,4 @@ -triton: +inferenceServer: replicas: 1 image: fastml/triton-torchgeo:26.04-py3-geometric # works for CMSSW run3 command: ["/bin/sh", "-c"] diff --git a/values/values-nautilus-atlas.yaml b/values/values-nautilus-atlas.yaml index 146963eb..abd51994 100644 --- a/values/values-nautilus-atlas.yaml +++ b/values/values-nautilus-atlas.yaml @@ -1,6 +1,6 @@ serverLoadThreshold: 100 -triton: +inferenceServer: name: triton-atlas image: milescb/traccc-aas:v1.1 affinity: diff --git a/values/values-nautilus-cms.yaml b/values/values-nautilus-cms.yaml index db519037..16d50cde 100644 --- a/values/values-nautilus-cms.yaml +++ b/values/values-nautilus-cms.yaml @@ -1,6 +1,6 @@ serverLoadThreshold: 100 -triton: +inferenceServer: replicas: 5 # image: fastml/triton-torchgeo:21.02-py3-geometric # run2 # image: fastml/triton-torchgeo:22.07-py3-geometric # run3 diff --git a/values/values-nautilus-icecube.yaml b/values/values-nautilus-icecube.yaml index 2686f64b..2e58b72e 100644 --- a/values/values-nautilus-icecube.yaml +++ b/values/values-nautilus-icecube.yaml @@ -1,6 +1,6 @@ serverLoadThreshold: 100 -triton: +inferenceServer: image: nvcr.io/nvidia/tritonserver:26.08-py3 affinity: nodeAffinity: diff --git a/values/values-nereid.yaml b/values/values-nereid.yaml new file mode 100644 index 00000000..363cba02 --- /dev/null +++ b/values/values-nereid.yaml @@ -0,0 +1,113 @@ +# Example deployment backed by Nereid instead of Triton. +# +# Nereid (https://github.com/ngpaladi/nereid-server) serves the same KServe v2 +# gRPC protocol as Triton, so Envoy, the Service, the ServiceMonitor and the +# autoscaling metric are all unchanged. What differs is the image, how models +# are configured, and the probe endpoints. +# +# Upstream features this file assumes (see the PR description): +# - the KServe v2 HTTP health endpoints used by the probes below +# (ngpaladi/nereid-server#19) +# - Triton-named nv_inference_* metrics on /metrics, which the default +# serverLoadMetric and the ServiceMonitor scrape (same PR) +# +# scaleFromZero additionally needs the RepositoryIndex RPC +# (ngpaladi/nereid-server#20). It is left disabled here, but nothing about it +# is Triton-specific: Nereid vendors the same `inference.GRPCInferenceService` +# proto, so the path Envoy routes and the Lua filter matches is identical. + +inferenceServer: + type: nereid + + # TODO: pin a released tag once nereid-server publishes one. The repository + # has no releases or tags yet and its GHCR package is not published, so this + # reference is a placeholder rather than a working image. + image: ghcr.io/ngpaladi/nereid-server:latest + + # Nereid is configured by nereid.yaml, not by flags: fall back to the image's + # own entrypoint. `null` (not `[]`) is what drops the chart's Triton defaults. + command: null + args: null + + replicas: 1 + + resources: + limits: { cpu: 1, memory: 2G } + requests: { cpu: 1, memory: 2G } + + # No HTTP port 8000: Nereid serves gRPC plus a single HTTP port carrying both + # /metrics and the health endpoints. Keeping Triton's port numbers means the + # rest of the chart needs no per-server special-casing. + service: + ports: + - { name: grpc, port: 8001, targetPort: 8001, protocol: TCP } + - { name: metrics, port: 8002, targetPort: 8002, protocol: TCP } + + nereid: + config: + server: + bind_addr: "[::]:8001" + http_addr: "[::]:8002" + # Absolute, because Nereid resolves this relative to its working + # directory while the models arrive on the modelRepository mount. + ml_backends_path: "/models" + # One entry per model folder under ml_backends_path. Nereid refuses to + # start with an empty list. + models: + - name: model3 + device: cpu + queue_capacity: 16 + + # Mount the model folders. A read-only volume serves .pt/ONNX/TensorFlow + # models fine, but not Nereid's Python backend: that one builds a venv/ inside + # each model folder at load time, and the container runs as uid 10001. + modelRepository: + enabled: true + storageType: "pvc" + mountPath: "/models" + pvc: + claimName: nereid-models + + # `command: null` drops the chart's Triton exec probe: the Nereid image has no + # curl, and setting a second handler without clearing the first is an error. + readinessProbe: + reset: false + command: null + httpGet: + path: /v2/health/ready + port: metrics + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 10 + timeoutSeconds: 15 + successThreshold: 1 + + startupProbe: + reset: false + httpGet: + path: /v2/health/live + port: metrics + initialDelaySeconds: 0 + periodSeconds: 10 + failureThreshold: 24 + timeoutSeconds: 15 + +envoy: + enabled: true + resources: + requests: + cpu: 0.1 + memory: "128Mi" + +prometheus: + enabled: false + +grafana: + enabled: false + +keda: + enabled: false + +metricsCollector: + # Queries nv_gpu_* series, which Nereid does not export. + enabled: false