From 8f96753560fe5c430a59971bb1161af69ec9bfcb Mon Sep 17 00:00:00 2001 From: Kevin Reeuwijk Date: Wed, 26 Aug 2026 12:49:40 +0200 Subject: [PATCH 1/3] NVIDIA Network Operator 26.7.0 --- .../nvidia-network-operator-26.7.0/README.md | 234 + .../charts/network-operator-26.7.0.tgz | Bin 0 -> 138803 bytes .../charts/network-operator/.helmignore | 23 + .../charts/network-operator/Chart.yaml | 26 + .../charts/network-operator/README.md | 274 + .../maintenance-operator-chart/.helmignore | 23 + .../maintenance-operator-chart/Chart.yaml | 6 + .../maintenance-operator-chart/README.md | 35 + ...nvidia.com_maintenanceoperatorconfigs.yaml | 93 + ...intenance.nvidia.com_nodemaintenances.yaml | 275 + .../templates/_helpers.tpl | 62 + .../templates/certificates.yaml | 43 + .../templates/deployment.yaml | 89 + .../templates/operatorconfig.yaml | 23 + .../templates/role.yaml | 154 + .../templates/role_binding.yaml | 36 + .../templates/serviceaccount.yaml | 12 + .../templates/webhook.yaml | 48 + .../maintenance-operator-chart/values.yaml | 114 + .../charts/node-feature-discovery/.helmignore | 23 + .../charts/node-feature-discovery/Chart.yaml | 18 + .../charts/node-feature-discovery/README.md | 394 ++ .../node-feature-discovery/README.md.gotmpl | 9 + .../_metadata-site.gotmpl | 15 + .../node-feature-discovery/_metadata.gotmpl | 11 + .../node-feature-discovery/_templates.gotmpl | 158 + .../crds/nfd-api-crds.yaml | 788 +++ .../templates/_helpers.tpl | 126 + .../templates/clusterrole.yaml | 162 + .../templates/clusterrolebinding.yaml | 72 + .../templates/master-networkpolicy.yaml | 26 + .../templates/master-pdb.yaml | 17 + .../templates/master.yaml | 181 + .../templates/nfd-gc-networkpolicy.yaml | 26 + .../templates/nfd-gc-pdb.yaml | 17 + .../templates/nfd-gc.yaml | 130 + .../templates/nfd-master-conf.yaml | 12 + .../templates/nfd-topologyupdater-conf.yaml | 12 + .../templates/nfd-worker-conf.yaml | 12 + .../templates/post-delete-job.yaml | 102 + .../templates/prometheus.yaml | 26 + .../templates/role.yaml | 25 + .../templates/rolebinding.yaml | 18 + .../templates/serviceaccount.yaml | 58 + .../templates/topologyupdater-crds.yaml | 278 + .../topologyupdater-networkpolicy.yaml | 26 + .../templates/topologyupdater.yaml | 198 + .../templates/worker-networkpolicy.yaml | 26 + .../templates/worker.yaml | 210 + .../node-feature-discovery/values.schema.json | 4622 +++++++++++++++++ .../charts/node-feature-discovery/values.yaml | 1261 +++++ .../charts/sriov-network-operator/.helmignore | 23 + .../charts/sriov-network-operator/Chart.yaml | 12 + .../charts/sriov-network-operator/README.md | 170 + ...networkplumbingwg.io_deviceattributes.yaml | 75 + ...rkplumbingwg.io_sriovresourcepolicies.yaml | 228 + ...sriovnetwork.openshift.io_ovsnetworks.yaml | 105 + ...vnetwork.openshift.io_sriovibnetworks.yaml | 78 + ...openshift.io_sriovnetworknodepolicies.yaml | 265 + ...k.openshift.io_sriovnetworknodestates.yaml | 529 ++ ....openshift.io_sriovnetworkpoolconfigs.yaml | 138 + ...iovnetwork.openshift.io_sriovnetworks.yaml | 136 + ...ork.openshift.io_sriovoperatorconfigs.yaml | 121 + .../templates/NOTES.txt | 6 + .../templates/_helpers.tpl | 62 + .../templates/certificate.yaml | 71 + .../templates/clusterrole.yaml | 60 + .../templates/clusterrolebinding.yaml | 29 + .../templates/configmap.yaml | 56 + .../templates/extra-list.yaml | 8 + .../templates/operator.yaml | 168 + .../templates/pre-delete-webooks.yaml | 65 + .../templates/role.yaml | 148 + .../templates/rolebinding.yaml | 31 + .../templates/serviceaccount.yaml | 15 + .../templates/sriovoperatorconfig.yaml | 31 + .../charts/sriov-network-operator/values.yaml | 160 + ...f.io_networkattachmentdefinitions_crd.yaml | 58 + .../crds/mellanox.com_hostdevicenetworks.yaml | 111 + .../crds/mellanox.com_ipoibnetworks.yaml | 83 + .../crds/mellanox.com_macvlannetworks.yaml | 97 + .../crds/mellanox.com_nicclusterpolicies.yaml | 2187 ++++++++ .../crds/mellanox.com_nicnodepolicies.yaml | 772 +++ .../network-operator/templates/NOTES.txt | 3 + .../network-operator/templates/_helpers.tpl | 86 + .../templates/admission_controller.yaml | 152 + .../templates/keep-ncp-hook.yaml | 94 + .../templates/nodefeaturerules.yaml | 16 + .../network-operator/templates/operator.yaml | 179 + .../network-operator/templates/role.yaml | 389 ++ .../templates/role_binding.yaml | 27 + .../templates/service_account.yaml | 20 + .../templates/upgrade-crd-hook.yaml | 105 + .../charts/network-operator/values.yaml | 349 ++ packs/nvidia-network-operator-26.7.0/logo.png | Bin 0 -> 7892 bytes .../nvidia-network-operator-26.7.0/pack.json | 17 + .../values.yaml | 388 ++ 97 files changed, 18552 insertions(+) create mode 100644 packs/nvidia-network-operator-26.7.0/README.md create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator-26.7.0.tgz create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/.helmignore create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/Chart.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/README.md create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/.helmignore create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/Chart.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/README.md create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/values.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/.helmignore create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/Chart.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md.gotmpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata-site.gotmpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata.gotmpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_templates.gotmpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/_helpers.tpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-networkpolicy.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-pdb.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-networkpolicy.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/prometheus.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/role.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-networkpolicy.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker-networkpolicy.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.schema.json create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/.helmignore create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/Chart.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/README.md create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_deviceattributes.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_sriovresourcepolicies.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/NOTES.txt create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/_helpers.tpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/certificate.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/configmap.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/extra-list.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/operator.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/role.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/values.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_hostdevicenetworks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_ipoibnetworks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_macvlannetworks.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicclusterpolicies.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicnodepolicies.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/NOTES.txt create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/_helpers.tpl create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/admission_controller.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/keep-ncp-hook.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/nodefeaturerules.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/operator.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role_binding.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/service_account.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/upgrade-crd-hook.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/charts/network-operator/values.yaml create mode 100644 packs/nvidia-network-operator-26.7.0/logo.png create mode 100644 packs/nvidia-network-operator-26.7.0/pack.json create mode 100644 packs/nvidia-network-operator-26.7.0/values.yaml diff --git a/packs/nvidia-network-operator-26.7.0/README.md b/packs/nvidia-network-operator-26.7.0/README.md new file mode 100644 index 00000000..69b449c1 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/README.md @@ -0,0 +1,234 @@ +# NVIDIA Network Operator + +The Nvidia Network Operator pack installs and manages the lifecycle of Nvidia Network Operator. It's a versatile operator that can facilitate many different networking scenarios. Nvidia Network Operator +leverages [Kubernetes CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) +and [Operator SDK](https://github.com/operator-framework/operator-sdk) to manage Networking related Components in order +to enable Fast networking, RDMA and GPUDirect for workloads in a Kubernetes cluster. Network Operator works in +conjunction with [GPU-Operator](https://github.com/NVIDIA/gpu-operator) to enable GPU-Direct RDMA on compatible systems. + +The Goal of Network Operator is to manage _all_ networking related components to enable execution of RDMA and GPUDirect +RDMA workloads in a kubernetes cluster including: + +* Mellanox Networking drivers to enable advanced features +* Kubernetes device plugins to provide hardware resources for fast network +* Kubernetes secondary network for Network intensive workloads + + +## Prerequisites + +* RDMA capable hardware: Mellanox ConnectX-5 NIC or newer. +* NVIDIA GPU and driver supporting GPUDirect e.g Quadro RTX 6000/8000 or newer. + (GPU-Direct only) +* Operating Systems: Ubuntu 20.04 LTS or newer. + +> __NOTE__: ConnectX-6 Lx is not supported. + + +## Parameters + +### dpf-operator-config +| **Parameter** | **Type** | **Default Value** | **Description** | +|---|---|---|---| +| nfd.enabled | bool | `true` | Deploy Node Feature Discovery operator | +| nfd.deployNodeFeatureRules | bool | `true` | | Deploy Node Feature Rules to label the nodes with the discovered features | +| upgradeCRDs | bool | `true` | Enable CRDs upgrade with helm pre-install and pre-upgrade hooks | +| sriovNetworkOperator.enabled | bool | `false` | Deploy SR-IOV Network Operator | +| maintenanceOperator.enabled | bool | `false` | Deploy Maintenance Operator | +| node-feature-discovery.enableNodeFeatureApi | bool | `true` | The Node Feature API enable communication between nfd master and worker through NodeFeature CRs. Otherwise communication is through gRPC | +| node-feature-discovery.featureGates.NodeFeatureAPI | bool | `true` | | +| node-feature-discovery.image.repository | bool | `"nvcr.io/nvidia/mellanox/node-feature-discovery"` | | +| node-feature-discovery.image.tag | bool | `"network-operator-v26.7.0"` | | +| node-feature-discovery.image.pullPolicy | bool | `"IfNotPresent"` | | +| node-feature-discovery.imagePullSecrets | list | `[]` | imagePullSecrets for node-feature-discovery Network Operator related images | +| node-feature-discovery.master.serviceAccount.name | string | `"node-feature-discovery"` | | +| node-feature-discovery.master.serviceAccount.create | bool | `true` | | +| node-feature-discovery.master.config.extraLabelNs | list | `["nvidia.com"]` | | +| node-feature-discovery.gc.enable | bool | `true` | Specifies whether the NFD Garbage Collector should be created | +| node-feature-discovery.gc.replicaCount | int | `1` | Specifies the number of replicas for the NFD Garbage Collector | +| node-feature-discovery.gc.serviceAccount.name | string | `"node-feature-discovery"` | The name of the service account for garbage collector to use. If not set and create is true, a name is generated using the fullname template and -gc suffix | +| node-feature-discovery.gc.serviceAccount.create | bool | `false` | | +| node-feature-discovery.worker.serviceAccount.name | string | `"node-feature-discovery"` | | +| node-feature-discovery.worker.serviceAccount.create | bool | `false` | Disable creation to avoid duplicate serviceaccount creation by master spec above | +| node-feature-discovery.worker.serviceAccount.tolerations | list | Tolerations for control plane taints and `nvidia.com/gpu:NoSchedule` | | +| node-feature-discovery.worker.serviceAccount.config.sources.pci.deviceClassWhitelist | list | `["0300","0302"]` | | +| node-feature-discovery.worker.serviceAccount.config.sources.pci.deviceLabelFields | list | `["vendor"]` | | +| sriov-network-operator.cniBinPath | string | `"/opt/cni/bin"` | Directory where SRIOV CNI binaries will be deployed on the node | +| sriov-network-operator.operator.resourcePrefix | string | `"nvidia.com"` | Prefix to be used for resources names | +| sriov-network-operator.operator.admissionControllers.enabled | bool | `false` | Enables admission controller | +| sriov-network-operator.operator.admissionControllers.certificates.secretNames.operator | string | `"operator-webhook-cert"` | | +| sriov-network-operator.operator.admissionControllers.certificates.secretNames.injector | string | `"network-resources-injector-cert"` | | +| sriov-network-operator.operator.admissionControllers.certificates.certManager.enabled | bool | `true` | When enabled, makes use of certificates managed by cert-manager | +| sriov-network-operator.operator.admissionControllers.certificates.certManager.generateSelfSigned | bool | `true` | When enabled, certificates are generated via cert-manager and then name will match the name of the secrets defined above | +| sriov-network-operator.operator.admissionControllers.certificates.custom.enabled | bool | `false` | If not specified, no secret is created and secrets with the names defined above are expected to exist in the cluster. In that case, the ca.crt must be base64 encoded twice since it ends up being an env variable | +| sriov-network-operator.operator.admissionControllers.certificates.custom.operator.caCrt | string | `""` | | +| sriov-network-operator.operator.admissionControllers.certificates.custom.operator.tlsCrt | string | `""` | | +| sriov-network-operator.operator.admissionControllers.certificates.custom.operator.tlsKey | string | `""` | | +| sriov-network-operator.operator.admissionControllers.certificates.custom.injector.caCrt | string | `""` | | +| sriov-network-operator.operator.admissionControllers.certificates.custom.injector.tlsCrt | string | `""` | | +| sriov-network-operator.operator.admissionControllers.certificates.custom.injector.tlsKey | string | `""` | | +| sriov-network-operator.images.operator | string | `"nvcr.io/nvidia/mellanox/sriov-network-operator:network-operator-v26.7.0"` | | +| sriov-network-operator.images.sriovConfigDaemon | string | `"nvcr.io/nvidia/mellanox/sriov-network-operator-config-daemon:network-operator-v26.7.0"` | | +| sriov-network-operator.images.sriovCni | string | `"nvcr.io/nvidia/mellanox/sriov-cni:network-operator-v26.7.0"` | | +| sriov-network-operator.images.ibSriovCni | string | `"nvcr.io/nvidia/mellanox/ib-sriov-cni:network-operator-v26.7.0"` | | +| sriov-network-operator.images.ovsCni | string | `"nvcr.io/nvidia/mellanox/ovs-cni-plugin:network-operator-v26.7.0` | | +| sriov-network-operator.images.rdmaCni | string | `"nvcr.io/nvidia/mellanox/rdma-cni:network-operator-v26.7.0"` | | +| sriov-network-operator.images.sriovDevicePlugin | string | `"nvcr.io/nvidia/mellanox/sriov-network-device-plugin:network-operator-v26.7.0"` | | +| sriov-network-operator.images.sriovDraDriver | string | `"nvcr.io/nvidia/mellanox/dra-driver-sriov:network-operator-v26.7.0"` | | +| sriov-network-operator.images.resourcesInjector | string | `"ghcr.io/k8snetworkplumbingwg/network-resources-injector:v1.7.0"` | | +| sriov-network-operator.images.webhook | string | `"nvcr.io/nvidia/mellanox/sriov-network-operator-webhook:network-operator-v26.7.0"` | | +| sriov-network-operator.imagePullSecrets | list | `[]` | imagePullSecrets for sriov-network-operator related images | +| sriov-network-operator.sriovOperatorConfig.deploy | bool | `true` | Deploy ``SriovOperatorConfig`` custom resource | +| sriov-network-operator.sriovOperatorConfig.configDaemonNodeSelector | map | `beta.kubernetes.io/os: "linux"` and `network.nvidia.com/operator.mofed.wait: "false"` | Selects the nodes to be configured | +| maintenance-operator-chart.operatorConfig.deploy | bool | `false` | Deploy MaintenanceOperatorConfig. Maintenance Operator might be already deployed on the cluster, in that case no need to deploy MaintenanceOperatorConfig | +| maintenance-operator.imagePullSecrets | list | `[]` | imagePullSecrets for maintenance-operator related images | +| maintenance-operator-chart.operator.image.repository | string | `"nvcr.io/nvidia/mellanox"` | | +| maintenance-operator-chart.operator.image.name | string | `"maintenance-operator"` | | +| maintenance-operator-chart.operator.image.tag | string | `"network-operator-v26.7.0"` | | +| maintenance-operator-chart.operator.admissionController.enable | bool | `false` | Enable admission controller of the operator | +| maintenance-operator-chart.operator.admissionController.certificates.secretNames.operator | string | `"maintenance-webhook-cert"` | Secret name containing certificates for the operator admission controller | +| maintenance-operator-chart.operator.admissionController.certificates.certManager.enable | bool | `false` | Use cert-manager for certificates | +| maintenance-operator-chart.operator.admissionController.certificates.certManager.generateSelfSigned | bool | `false` | Generate self-signed certificates with cert-manager | +| maintenance-operator-chart.operator.admissionController.certificates.custom.enable | bool | `false` | Enable custom certificates using secrets | +| operator.resources.limits.cpu | string | `"500m" | | +| operator.resources.limits.memory | string | `"128Mi" | | +| operator.resources.requests.cpu | string | `"5m" | | +| operator.resources.requests.memory | string | `"64Mi" | | +| operator.tolerations | list | Tolerations for control plane taints | | +| operator.nodeSelector | map | `{}` | Configure node selector settings for the operator | +| operator.affinity | map | Affinity for control plane nodes | Configure node affinity settings for the operator | +| operator.repository | string | `"nvcr.io/nvidia/cloud-native"` | Network Operator image repository | +| operator.image | string | `"network-operator"` | Network Operator image name | +| operator.imagePullSecrets | list | `[]` | imagePullSecrets for Network Operator related images | +| operator.nameOverride | string | `""` | Name to be used as part of objects name generation | +| operator.fullnameOverride | string | `""` | Name to be used to replace generated names | +| operator.tag | string | `""` | If defined will use the given image tag, else chart AppVersion will be used | +| operator.cniBinDirectory | string | `"/opt/cni/bin"` | Directory where CNI binaries will be deployed on the node | +| operator.cniNetworkDirectory | string | `"/etc/cni/net.d"` | Directory where CNI network configuration will be deployed on the nodes | +| operator.maintenanceOperator.useRequestor | bool | `false` | Enable the use of maintenance operator upgrade logic | +| operator.maintenanceOperator.requestorID | string | `"nvidia.network.operator"` | | +| operator.maintenanceOperator.nodeMaintenanceNamePrefix | string | `"nvidia.network.operator"` | | +| operator.maintenanceOperator.nodeMaintenanceNamespace | string | `"default"` | | +| operator.useDTK | bool | `true` | Enable the use of Driver ToolKit to compile DOCA-OFED Drivers (OpenShift only) | +| operator.admissionController.enable | bool | `false` | Deploy with admission controller | +| operator.admissionController.useCertManager | bool | `false` | Use cert-manager for generating self-signed certificate | +| operator.admissionController.certificate.caCrt | string | `""` | Custom certificate instead of using cert-manager| +| operator.admissionController.certificate.tlsCrt | string | `""` | Custom certificate instead of using cert-manager| +| operator.admissionController.certificate.tlsKey | string | `""` | Custom certificate instead of using cert-manager| +| operator.ofedDriver.initContainer.enable | bool | `true` | Deploy OFED init container | +| operator.ofedDriver.initContainer.repository | string | `"nvcr.io/nvidia/mellanox"` | OFED init container image repository | +| operator.ofedDriver.initContainer.image | string | `"network-operator-init-container"` | OFED init container image name | +| operator.ofedDriver.initContainer.version | string | `"network-operator-v26.7.0"` | OFED init container image version | +| imagePullSecrets | list | `[]` | An optional list of references to secrets to use for pulling any of the Network Operator images | + + +## Upgrade + +### Upgrade CRDs to compatible version + +The network-operator helm chart contains a hook(pre-install, pre-upgrade) that will automatically upgrade required CRDs in the cluster. +The hook is enabled by default. If you don't want to upgrade CRDs with helm automatically, +you can disable auto upgrade by setting `upgradeCRDs: false` in the helm chart values. +Then you can follow the guide below to download and apply CRDs for the concrete version of the network-operator. + + +## Usage + +#### Deploy Network Operator without Node Feature Discovery + +By default the network operator +deploys [Node Feature Discovery (NFD)](https://github.com/kubernetes-sigs/node-feature-discovery) +in order to perform node labeling in the cluster to allow proper scheduling of Network Operator resources. If the nodes +where already labeled by other means (either deployed from upstream or deployed within another deployment), it is possible to disable the deployment of NFD by setting +`nfd.enabled=false` chart parameter and make sure that the installed version is `v0.13.2` or newer and has NodeFeatureApi enabled. + +##### Deploy NFD from upstream with NodeFeatureApi enabled +``` +$ export NFD_NS=node-feature-discovery +$ helm repo add nfd https://kubernetes-sigs.github.io/node-feature-discovery/charts +$ helm repo update +$ helm install nfd/node-feature-discovery --namespace $NFD_NS --create-namespace --generate-name --set enableNodeFeatureApi='true' +``` +For additional information , refer to the official [NVD deployment with Helm](https://kubernetes-sigs.github.io/node-feature-discovery/v0.13/deployment/helm.html) + +##### Deploy Network Operator without Node Feature Discovery +``` +$ helm install --set nfd.enabled=false -n network-operator --create-namespace --wait network-operator nvidia/network-operator +``` + +##### Currently the following NFD labels are used: + +| Label | Where | +|-----------------------------------------------|---------------------------------------------------| +| `feature.node.kubernetes.io/pci-15b3.present` | Nodes bearing Nvidia Mellanox Networking hardware | +| `nvidia.com/gpu.present` | Nodes bearing Nvidia GPU hardware | + +> __Note:__ The labels which Network Operator depends on may change between releases. + +> __Note:__ By default the operator is deployed without an instance of `NicClusterPolicy` and `MacvlanNetwork` +> custom resources. The user is required to create it later with configuration matching the cluster or use chart parameters to deploy it together with the operator. + +#### Deploy Network Operator with Admission Controller + +The Admission Controller can be optionally included as part of the Network Operator installation process. +It has the capability to validate supported Custom Resource Definitions (CRDs), which currently include NicClusterPolicy and HostDeviceNetwork. +By default, the deployment of the admission controller is disabled. To enable it, you must set `operator.admissionController.enabled` to `true`. + +Enabling the admission controller provides you with two options for managing certificates. +You can either utilize [cert-manager](https://cert-manager.io/docs/installation/) for generating a self-signed certificate automatically, or you can provide your own self-signed certificate. + +To use `cert-manager`, ensure that `operator.admissionController.useCertManager` is set to `true`. Additionally, make sure that you deploy cert-manager before initiating the Network Operator deployment. + +If you prefer not to use `cert-manager`, set `operator.admissionController.useCertManager` to `false`, and then provide your custom certificate and key using `operator.admissionController.certificate.tlsCrt` and `operator.admissionController.certificate.tlsKey`. + +> __NOTE__: When using your own certificate, the certificate must be valid for -webhook-service.< +> Release_Namespace>.svc, e.g. network-operator-webhook-service.network-operator.svc + +> __NOTE__: When deploying network operator with admission controller using helm, you need to append `--wait` to helm install and helm upgrade commands +> + +##### Generating self-signed certificate using OpenSSL + +To generate a self-signed SSL certificate valid for a specific hostname, you can use the `openssl` command-line tool. +First, navigate to the directory where you want to store your certificate and key files. Then, run the following +command: + +```bash +SVCNAME="network-operator-webhook-service.network-operator.svc" +openssl req -x509 -nodes -batch -newkey rsa:2048 -keyout server.key -out server.crt -days 365 -addext "subjectAltName=DNS:$SVCNAME" +``` + +Replace `SVCNAME` with the SVC name follows this convention -webhook-service..svc. +This command will generate a new RSA key pair with 2048 bits and create a self-signed certificate (`server.crt`) and +private key (`server.key`) that are valid for 365 days. + + +## Additional components + +### Node Feature Discovery + +Nvidia Network Operator relies on the existance of specific node labels to operate properly. e.g label a node as having +Nvidia networking hardware available. This can be achieved by either manually labeling Kubernetes nodes or using +[Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery) to perform the labeling. + +To allow zero touch deployment of the Operator we provide a helm chart to be used to optionally deploy Node Feature +Discovery in the cluster. This is enabled via `nfd.enabled` chart parameter. + +### SR-IOV Network Operator + +Nvidia Network Operator can operate in unison with SR-IOV Network Operator to enable SR-IOV workloads in a Kubernetes +cluster. We provide a helm chart to be used to optionally +deploy [SR-IOV Network Operator](https://github.com/k8snetworkplumbingwg/sriov-network-operator) in the cluster. This is +enabled via `sriovNetworkOperator.enabled` chart parameter. + +SR-IOV Network Operator can work in conjuction with [IB Kubernetes](#ib-kubernetes) to use InfiniBand PKEY Membership +Types + +For more information on how to configure SR-IOV in your Kubernetes cluster using SR-IOV Network Operator refer to the +project's github. + + +## References + +- [Official Documentation](https://docs.nvidia.com/networking/software/cloud-orchestration/index.html) +- [Operator SDK](https://github.com/operator-framework/operator-sdk) +- [GPU-Operator](https://github.com/NVIDIA/gpu-operator) \ No newline at end of file diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator-26.7.0.tgz b/packs/nvidia-network-operator-26.7.0/charts/network-operator-26.7.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..4a7e10d1ea094c75a2618777afa3e248133bc757 GIT binary patch literal 138803 zcmV)lK%c)KiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PMa0f7`gRD0+U@UxDZBy|K3@ZRgu=eVRR2k)2d0iS61>_VjM| z#SjTes7ZhYKsnl`?{9w}3;=#7McGa^yA^&&ED{6;gTY{CFf(98-EeX3@q`GhxY!%b zu~7aJ$I*Y>SJ~g+-+%Vx3H-mmzhC|T;NZ#A|2TO1=*i=!kG^~K-P8ZrKX~%=@&13H z{doE@UYqXABm!p~>^cmDn$Bne?5VF4xbaNwbU zvydvN!lt257@kBV93aGSOb!v_A@QaJt5gs#q%zc=)TMJPg z4z*0kA;L)#(E#f~4zlrE3i`E)d{bIWjsGujloENb8$dVzfBfk2lSftl|J`>78~*<# z$^$gw$x_hST%kw%kM_~Y`|;6uICwC4fZou6FiAp`vXBU*<^&BB9L$M(vx_W;L67`> zw5=;_*;iXT|2cSomOMoVsh#>MYAc;bhp@7FpL^WqcH&o4`8{3FK zc!2(Do8S|LDMJ_qJXs<>E!IO=0l>Ow)^)hIcXM;&V*u9YVzw8VigNGGcyxSnaqKpc#g9RBpKp~Y% z&`GLF$a2t8S*pMp!VGNq$H3caS~xg zq(2z2X?Qq5=mGLPbVQPfFVP7P33_Qn4|L>250t}#klol1krFBC`use#a71`QA{e@s zY-CQPPPWklF&|0YLkNxOwro?-lI?H+{T%}kXmoxgQJTyI4lV7^Nfe_*5RWpca1`m8 zfZwjxobzkx5A09fK&jy~K_n3if-w3siDECt49`f2PG~TSQmKeI;}H#(C`hH^F%raV z`Y23=CIXOX97xh1ToaO}iqURY}(tm_h>K6P#vm~2=csVC!lF_Z2-ZfK; zQ)cBMG$HDS5T?at6k|ON0FXuy5zw5P3!cvAMHtcOT>9u#FY_BJt4&gweKR{h8|64D z4?LrDJB(L)@#qnt3OdRQ!V)NuzC=MpFiR73GbaqC%+jW&my?%A1B4PT)e(t^A|q2X zn-+vO!mMloVYfNu$Wn_aPcjPq>2Rz!Z!M z069ye=z;`-NM*QHV|O*;MS^Iq6PiY(Zm-;&l85N0p9cXc~}V5b%^K^K7NArAr_PRwSEPWC3t3kdI2> zH+qIoH1u0$DzvP>_45FsSzs!{fG*6&p}F6jz{aB)pdRKcEGBv*8SyA0fyR}b^E3)I z`!)cC(2OC@0x*?!fxo8lgb2O23j$lP%yzEEt{heAsv|%SI_N-IwrDl(oCIO3B5dkH zhcnwzkc~-kEx!0!L*prkG?FM-$Y!9hCPhki5jI^=iDrap64GLU)C)#;IZdNTzt388 zyBf6R%>pFT>6Cu-`%g6t&GIb+jph-GBfQ`=L}6-lvuwC-v{}{3($ZYkpq3R7p72FB zY)4OKZ5K%0@G%r(5_6_;vIeD_Z@Vb{!Tk&j29z}k16zM7xuGMZctp(jEIakkHPQM% zv?F*#e64{CMzpj-`JRDd%Qoov*CEg?gg1nNuUnV)(O7|<|Irbvem2ofHNg{ z%oWlzy}l zY^vqgDeNE?13+%lx4g?dG@t0n&3SO%pxumdXytQcr#d3Sx3$_<7s|gHWglfosR&V) z)QM19N5};Qvo!TCccMWrqO?F?*^YceCR$4MbVJ3tq3rL#O!Y7=CNrdX?q%gG{p&5r zu%g&4pyfv`jo2aVq8MKj2|6a9md1db3_MxFb1M@IztCP)Dd^~@)U?)Fat^QS+Fjy!wV8kFX)Wb;0CDO4rMpka2GLd z+t&M%-I@VJ*JGyG0EHHsD}e%oCLa?LD5DY+%?q?*tUnEmmf@qZ{*4t1uq3;MwxAOB z1EElyN~PDx1WWSl2_h`up>Fa<@5hpAk)J9=Sg4gGG$DHb#Cj?gnWC8E#V9m~O_N%r z?gyy4X3aB%0z4Ax5Pd2=M99ocd{ly>O{zcx5{_@wO_ka1ztK*yh9-Utv zpS>OKzpAw8`+gHN$0tYKu&7A>p2qT8Z%7?uIUb?2^YQ!Pao^B(Yy1p(Uh9y6tJ*zMwkMz!|1SvDD(ZhO(Aa)eLD4U_+|v+-x25p>o)Hh z+0io^MW4+&aea5UszzV< z&e*4Fb{P_pK*@sD+o&^6vA0H@F2y4#tp5`p64PY98%~JA6&04tL$no9mVR^^fJL!i zP)9Rke$1yN^lvcLYAncjTN(SbDw8o-8p8{p8N(GwxhLZh8WoKc`CKTDbekw;iblD; zAd2!cqL)9S1PdG!MTAiTWEuoS%4r%!_WVdI9AwHxjx!B~A%LvauHi8ncx5|Ed8mU2 zjmGg74Yu8Xp~K={CLm+SY3wtG>4$}ss&mjVfI`t?9n-I zBl}e!qtSVZ)^h_^b(O&?Z5(1x3vgAtD44%{8Nf>>sisM@6;>O=8x;SNA(XUR6`U$s z>C>5TZ*47reyt7)DzEMmQ#gJs>DA8an@KQDP&S((ia2YwM0D8{Qc21xyoRI*T1~$! zqa$m`Dd|X(XzEE&)5~K7K2+>$g^FM9rmpT5C)G0gQh$aGT3x?&7=s7s6?7ieN#F1i zIL|Fi4T%VxZr~{5H?pvns~%2hDtN3FcpNDrm_|;-t-}zn9{-(!w-k=hRdz(F*)Nd_ zPN&V;v_yY$j!NZ?B;DHl{NQ4nF-uv!ZCL3DK` zmr{|~1G{k`sGC59IBsrp*r8AT!Ay%fvtE+yS_tll#D4no@W}HZ*JMz&5F|DoRkhVvqOo`lk^v9=9 z^vop|U0_^3F^+=`ILVH6iHIpPQUQz~B}7mjcH{Z2HD=Agu%;_wX7P{+a-n!~5fPGH zkbtvLUR`OxRNm7o-O(BJW2+pZ$NPg!9D_%CaY@~Zo};YY9Dx_(SC_}EFE5oI%I_hwEbVrMYnp+plh({t~DVl(Fz&`bqBIyVB55m;YmD-eOR;iEsI?K2*} zp;HnpgNVTJhk+jKjt{5xileKm{j00enY*5@8oCbAqo+A86vCnL-NaGx90n$6fd%EM zL`Uv0O6J1IWVyzwzQ+Tef225a&w+_rH&OIi&wFmsDiisVAVIDG`E%M?k-Jgp|$3Gsd&W#~(?M zf=_O?8+z!5Xi?UkBNv+^EFi`G78(2fv2no4$s{l95lmk4NgT7XO% zyOHwN1s`o~p`WuuWFY8TyPA~&$eHkj!xenpC66Sck>Dh9;GqS>W)+*)$);39kk@Wqle?_8tm5&D_Si8 zW^j6QE}Sr8sWnoi1$RFAJE-m;U|T`bp2F)#rwbwk4IMLRY)x_Ss?b|se!7(%TRQiJ zT1yJgb`hOs$3<|+WxPHKn$ZPeISO_Wi6lWL*aQu;r~~tn1;lsFW-2OpxO1`G)vCQ9 zXmm106Uwjv*A)9{Rrv%125dR7sZF_axgu=_Q!2r=14Q(ztL936z~~Fg&aj$aT|tAH zO~6MdToI$_>Rv3AD0JW#d}MHBbc5{h%~l>5#|QU;J6?0LDiSGEG=>e+0FvS65bg0q z?FEePxpvx6=S1|9guDWe&|rOhuBqN*v>&WDPP?d!Vpcm!> zob#%T_>2ZV_+2%6BPB-%br2)mfmsY0AZOW2e&LM~rt z*5FnfQqf=ttGeAerQsMJzJAwaYH|A&SSlI~C%&7wQNfH;v|bnL=Ju|JL)U(Xl=pi; zSw7)geK6T{!1^13yLOn``ylo9ho~hmR~*vwDGBW%l`+&Zszz?tDH%#Fnd=JqzT{VI zq6G&W<9<`B-6+qZwJgHzt!n=2+G`POJq>(r=vvvNjo)*vX5oCT24@p}Fl2~3yIn-3 zGM>Cv2Qk(QYi2k8w6Nrp)VPvk=AwEA4_btG=?|LSt{Z7`AA1mU&_Yb~%T#5gC_UQ4WP5j@ND6fdBueMC`&%IVB zOD&4C%!i#v5x;?h8y?4)h4OGP_y%346B4Khc?|vYpU6Ka5y6u9nWT&U^)ECd3ha%W zh50p8s>!p};HF|InM0KA2EHL9JFH%B*}*0EbP1v@ORuGnWPBAGS#4Gin>`7I_r z2S3AEz@a#5^VmnM3{vc*XzovX?nJug)LiN zCX~0&%tQ5j+N)zK;@$l1*#D>O&|vEiTWIUU*81y>$1!I%>bM7}rlNs$PNLYC^F8C! zrln8Hf+!6;hUgn0Ix4!e&o{Oszd%iP^o65~fBkEKlcaK3mg&w~9AGid3*s zZ^gzid6VT~NwB`K(W_0^bjPv%@mU?N*9eR8P3y#);T(SRSRj0y1>~@%A~X)8=Avh% z(jMc<{;0N*k>5A9)(@W>(?ti@t9@hZ3e08=+^DU3y>Ikf?wk|9zB_ob(f__g@jS1U(KQ_ZxWssdwHHzc9t^H23lEXqkp?kQIK&DcW-0Un9wKM};a6K;Yt^xwJ5VjT)dsQx_>pU|5IzLMAg8cdrk5 z`)y77auM81Bfg8jj=1q?1y@*cPNrtU4QY(wp*dUcKuk0>$FClB^>e@eLsSHdn7IAl z!m_Yq->itWCVrh|AqznZ|NK`AP26R#$*Kx6qf%M_w(IX@X`!2PId@k=G;>)6;-LvL?3zq_NIcz?Ks@)!J!KHT!VD&YW+-PyU1{z$J~^#-9^gz z$M)MkpWd3fSAuGvLK@tR0M5yRk{e5bmfD;pA*JRn)a$(AT zZQInQeE!m5|Gg$8@mP>}dO_UfLV#8NKhGY0SF!)@A3Wap|9p{B(S2>gZ^y8^OuRNt zYqO=d{cJGzEbz6Q0J@f!k($I!`m{UTHR*Z^`O$X^xzTGiy%2Izdx_m#4PQ@#fRK=c zyC!JbGb*4>7h7+`1y*kDZ`Cj&COGhMs^(nKUz)8DHtYI1k6KLPw?&lTkY@F3MOC|y z(kP}>go{=&7JShsg<4zRnrW$F`vqkoWwZYCbsw`M0~`@=V!Qr&XGf+cCdo`t1v6zsJfAZwJGsOF zP@I6ARJ$fYm*Nm^H;nId4R!sYfC6oY28YKH@op97mGASz~}Fce)O zV3fyrZ8hnAc|4grO=$E+LjU^LVbe>6XXsPa$Fk(%wp9h>to_anS0o8i$iKsxA|F*b z-9&&BJfRU)l++SYgn}p4XBsubH*ZQ$MaqWqgtK$bRV{}%GMTB)1YOXG%*e6cq_Me+ zubR3GVAX5*Oza({jsEu~N=5%;)6j3ez^ZCW(>1TZm!70ED?(ed zqLGp+pEpTN5a;SK8b}Z5E8FTb8C9X%GS^SDzm%@ zOW9O;|Iz;b)-KZjf2TkC68xZJ{b$?4UGIWi;s5va*|R6r_^-#0A8*$GmnfU~uTA{d zo#MYro;gV(b4B?myZY>I#+CciJ=b&cT+i*vU0G&wu$7%lVfT1b+N!!oQ4c)HNAt{{ z|8(lZHDiJFG)1!1oZBcm*eA2yaFWtUm2()n^3@mw zZt2^ZH;_5>=NQZISn}P^OVg=+sX{##poN$UBjBPw2$!#MJul_PVVdlaZ#B)9M%#wJ zTGyr4997M@=PH&E5bJ)M^%1B0riA1*9`{hb}OUv4pS8IBJtbO-HJO-PWxxo*^wF41w* zDr^F2^++46Q6Ac>C!5D&R#|hN7ZDs1;lW;4ctzNvYTPn2b#XboJpS-@crt!@d~x~V z#qi?z!_oM>GEA3+SOY<)q;MT&xwW&?qYo#;x5pP}!_je58!tsU_q{0)PM?K3G`us^ z=tLzylkOUUI6XT)A6}lG10*d2X*Th@a=Yc6Ly%{dZNsB$sTqy%_3+j4htZqk;rWN* z%gf{Q4=>-myZCcUdk1y+YgCmte|$20@#gr$kH;_me0uu&qNUBg-bpzwng{bvQ`F__ zHF`0*3~LRx&Pq9pYTKwsC*uz<#wQ)r-2_X$@m{FzfVG+lwF};p#wbU2liulgs0i;mPRuL&o#W+CEx?T6%Qfys#Vl zKR^D3g%3 zjD*+&u#T934l4Y&Ib)9Kf-oZGnc$Pkw{xYES434Cw2>5Zf+IEmrTP-I>hJfzKPbJ> z+aATy5sC0JyMyK6X<5N3-&XTzzwlNoYV=lM!u@e~fwB|GgZ+xu)GPqxX)jAeR`XIs z?bfk_M=MmcR*D5mdnbuFM9-c+ef-E7=DIHnYP8;TeNs?rAMclKY@gQxVyqu!L-<-+ zQkTJl#EDuSQFjMTBj)d9-0+%g6DBUcNEH^fZ+S=#(Ua2JSQSLy0rh7%4DLApg{xog zdiMJttIvNQf45)F|M~36qmBQ^7b#8WzfaxyFWg7*n>qouH)yPN1`OY9PJwTG3XIU+ z9vZ>Dec3H2lV#x^2yaH1kB&~!$>}9JJ{n&(oE+aFl)||vU(%RJh2!K9u{4Tqb<*tO z7AjE-p1IzB69y5cv3(hwoj&%;fVf1M4^^cSf%E7ez~XTj>fP#8Y?|3|0p+_g;#h6uLS-y4&(6Wj41d& zqSE}4U=_^auk7+i58c4SHL=(ViEX%7b8tyf`SUx0+?8yk-g|d0i_ec>oKW&n5eCb* z6-skGta}LQ6X}gEg~S1}XSQ zx{@7q_UbNQXG9e)vTfiyG$?(LiP7$@aVgDyPN|5s&|n*>KCXREnId9}1H#~rx%};y zK>eMOP~Ew5d^EhNpr)fnOZz<*+kK6zu=^IqRDN0=PxIThHk&6tuhc-ljGo%E;@h!QA9!9{ZAFyEB*o6~8u~Img z>#f53k~?t0&f87$JS2s>UoT*tD`C}aRRWPKT{Y%y@(@ipos^!$I9NoOm7X=mFE>4| ztoKSzmD;n$@qUi^SQ9l>HeJF}^<@QN4J^KjqdS*kp` z)g-h>iMI)$f=9$=0Fc$^R&S}+aIZe^f_C7x4#52^7$nM%is;S0jc9FmD1ljn!ySCI zZN-J0x`{hb3!CWPG!vNflYld=ozt4EkjjQ>l?e%yNaK&4bd}W-m`0kKj1c(wr|NFt zt<4PY*xlV$7~HPyGBpF73>{mPk_8gm%|KL%pxlkW#*0S9s^6^Rck zXD5eWENM~C1V#6`{yX@;MZ*V+hud!ftN8!pgM%vnfA)BD|H~IC8~(rH|MwRE{aI2( z_Kd^+wHMNFv~=))7sefk@NOw!SMdL5Po6!i^Z!l$k1tXx&X9J_N9RZKOXq;yq<-zj z&u=>8Yl~w_W9HW`_mrepXC^H@RMa#ayrDc|e7uVGugfOUYtKZlI7yi|&rzQ=gK zs`R{{z0~x7UyBcP#<(E&)yJy*zfT`Ot;Byl`R?&1{_BgB2j~ndMFf*z+=O{YH*>-u z<5&oGK^$D;8Ik_r0lJ)1iDa5UrY4ITn&99%O~52A2jBYgCK-I|Pw@5NTVKWc zZ!YL;@a_LMc!1txL3t|C_~=*;{6uiAOAUM)61-=s3;y?kzmNeB$=+{)P1M%^`SI}R z?Xe$+_t!`F`hUE?|7gFu{`a45&VRm0d4ODIZ9@pi?Xc+2B#IG8hJ(TCIw%o*K|>-D zW{6<9L^pV8&Pzaxg=9b`x&#gpvaD%{PaO!B7Evaq4h9ce;cEqI5D^m;F40e~Gj$D( z&X45J+jFH7dAPS%NrB1(xfgI2kVMHnFjR$JUS$s_)XU6Qd&WNH95Y-nFLO72&CUf9EZ>dMg;`9tK;jl3?5gAvO&Pv-%|!ooQ7-v z^y=)L=jt_%*_IYF;Dj(}RK@KIU7 z+y@**9}1Yr@E7Fx%ljuENsz*L__P6K19()y*>w$PRym|>b~qS(i{4u2;^Hit)VRg2 zak$VDY3MS~%fYwB=`jAsD2dV;HJh_tVQ7v;c!LGWZKp7^(*gyQn=;7-u~=p+OxJg4 z1ld^X4LcjBUUv`B5f9RsFk`103|?}9Vl5p|HsvBVyLbZTEwrGLDp=QiI;8={5&9`C zexB2YUTXzE!@Yc__S|vmUJ&s#^tcG-#D!e$Q5KSq{#?b;j^PW#Fa)zOjtWz2AV1+D zK{m-hI?6PTRyHO`M2XbIr-wj3QggAnritvpRBJKTqNFFo5Hrz}k`NIseMJ13d5^HE zj3t`m1!c2AnZIdr$mS3)FpV@l_(rx0Fhdi9a4@H2VNRp%L2ry%ilb<0I?;_3rWo7@ z5f>ECoAl7$1}{-esp~nu0czX)Ielg<$_FZa?D}1(f}~{eVsi+cuDF zDhV31DP{D9=Im#$kN=9^l6XRdoYQ1*xlD*`;cc9uIls}3xZuBR+9*Ryo{EO;(=5PN z0uW?MfDF`}46Od}P$D}Mz}Nng(%||+f!qaj37a4~H~uLm{CV)L*{r~*7Uf&QVZQf_ zIAbJG|KmMHC*zT34UF7qOn+;=d-N^`yPesZoK~ly&qW5!)c%svvL#D~naEJ+Idb?NLkg4@+91O0mt_I(r zAQchvrt;zq@;I6i^}!w)A;%d6zVQuBrjiHyQS4skURhdH)Nux zAm?`OSZv2iH+EC0v3m^_NEhgwBz*9V!Kfe!*Kpc94^XL9HVaB34g6X9(#8!8@>$I~ zwAI{ndaOowV9_L45idJkM4oqpsj62seb=qYg9qq6B{z02C1EZ+4!&_0Lp1h1Bg*6@ zf!TV{fjU?;JXNjY*NbI#LzHTY~=d!^y?-mcxhIs-A|ZZW}O{n^`)eyoIdx07bHCy;s&Y zn}#iIAg{2tzcH{kFMF3{2_7QPlSEkrl(F*sp%N*12uSJRv8_z>o(a? zdg2MFVJ~mgFi%kFs`I`kYcVrl=A(d_8bN<9{+e;e2WXTEL70k`fYE}~@$$$ikXrN8 zlKbIc@Cm_vHK=d-gnk5OvQGIl_~flu)~yfU3_hVNJL}-mQ1O(yHm zgKpuw8vJ10>55yE9*vgkbU&HfyOtke5$Fvh!kX2?nlB|j$&PbMnTtE6AIUM9L(>=*r!$dIi2sdOOQ-$`Me8SZLK*GcTLp zC~QI>9C{f~3_U>DcC>~cg+vTYmu_o=T`!v2LR7rIHVmK;TC3DHbs2)okM*}@wC!;gxlL375 zH6e+WPRz_fKImODw@_K{Wmmh^Il{#q3MM6n7#8bmQBea+;IB_r;VXG;~=7QJ!nq@V2Hm&L~_kiCS;8Q1g>SBit`dyMV|;B5Gj3x z24e+cIPfm#nFR&|o5GOfwiPro%-DegV-I>|iZk#pNSS^72=r)S^^YnoK%2SwGtM;XeM)Vhgekvy7%cbo~ zdOmYzS)ARy9q7*L2ZlXL5>2&m&PZ779^q7RtpEg?n0Aq6K^homY6iOzzhP}n_>eVM zGdsQYUlpOd+C_xr+F>{1Q%OcdsJHf0J)L^C@@XiX^Jq{js;36q${0luknwA_8RC_j z!6vEA(ilwlcnS?BMkUglvuYmSYJ#ecOAp9s)ZhuD!KonIiV&q~Vj;RFOKVf^?yxvF zDv~3itVFoZSFg$Ps$zHj5w=_aAft)GS7xmifrb?~A%>9vN&d-t!GAcxG5Mi%3mgrI z|EIwZs8|Eo^bfvV1iOg%Gr#WYU2UO$73d;Bs9_vtzV%qD#wW9O8<)5l{2VNf#hn@_ zdXqv|Wd+6I_-5Tk<=_XWD!$4WYui#ay`Lt8U0l2Y>B3n6D(m~=;!Wwp z9RD~!#hi1g^j5f=EqD`>OMp(8l+l#~(~Br0NO2zdgO^$c*+mR5=uEe5tp%C5VZ}+{ zftbOvk%|jP*$pHO-d3QBuw5(D7Ifu7=3%RM)CrdJ!NvR0$?)y*^P2y4FKXBt*x_k~ z2YDZ#?*9>a=FrrGun*)ha-#tg5+6R=fAT%@uE|m>UM4rcuitrv-+@rb3-MB-$IqT3 z4~HT7sL+;;uNg+*hW`BMMT9hhe!$s+FxbFX zt)%8<|cV|4;TWM28g=f#}pcy&^hG`qHTt4TC zk|>F;@*4`DXLk`^?V77U;f(6a-N$i-v?Nfhq$$L-(tc0j^MflNU1%oz?Y9@>SMM&) z55E1@F(0?pTcK7zo<^yhYi^>|-a%p7Qw9bJXG^gz-cBq?c<7^3WA{&RWVS{3RWyN$vr=^`Mq9DL1oOP_VtQ!(V20S-m z5F`!9u9?G{9lBBFY5*1Vb~{ZHVX&ZV7vudhxlN?_R~p*@?iZ2AG5ifabI0VbM) zLriWVgHtLHp*ee~G6bZ>0RqvJRyiQ4O==540{^IWZGN z2}%~kS|2QOow3Cj=?v6d*3h-2=)lmL#9C#25vw#&`{+U7E6-qU8QZ6cn(2-dVhR=F+M%{0eN1^6c+CP zkczqt4&c=nSZ#4-^jTSDPVOu&Gy}8ao@Q*`w59LbP$1X`cDZ6%=X*HEYjJym)-(GK zIuj((yvulU0O@9=rv<8PL`QCZn`%}$ZIQ!i`mohk&fW8T?tZWzdD$b#+hHI-4yi&{ zjTNp;P99v?P^QK%AaZsBLm$Gt6lm0$=rY(H6KFD(^*&N-9Gy399vo;U1Pas0&Su?WRnsz^M~*u>(YmDEt#yGauAvv?OL^A*+VKdR@?}dSo7<~o zvHxIeHgDd{-0+DkM}q6KVPCeXi5F{VDRA1x!)e#J@?y# z-nrj2z-YbXZ~Bw_$o}@=9bSvqz(v!Mkg3H?Xw-~-uu zz-C}c+ISTzElq(D8@dKc<)I0|VFWq9{fN)BUaEg5^QF{(mwIt|!+rF&>gJ!@8L&ER zIstEtjfm4oQLQ9+dbke@otfMb9L&Il$HsxQp8cY*jH+2NN;9CR$ue7@HexVj>4Jm~ z`@k8&vB!8t)zWGx?Bd-pvtas=0t$Ro<~)r;V{#S>2@>!b15=V|Ub@(YYO9bkP{D)= zh9l>}VZ23`ND$o*N?D;rtMKPDQcnR=m<`8O0v^XHqiVSa=19#`xsZDx z^HdwMnUVZcnNN;JBxT^O3HzwCJGh%Ct?7rBv$B{X(@9K~+31Lb7)T$Dj0e&rvvc@p z%+LtOBpP8!ic|Ahrb1&}dK$g`C!?ATs{BE1*_Bf;HP2obbJJg!7Febf^Jp4?Yv&8d*n`w%#|y-me6<~Tv0Qq zRme3E>jYPkYx>PK0tXw@l7dsYqGki}t6kohW8S?f4Q!?W?Ad-rUUsY?)i z@+!OCZ6)j`fx6Q4y5e<*I9+LZg>YS|{h8&t;&rwy5=8#EE>(9=&+TRG2AR5Le%6q% z8zt&Wvvv8pQlTVmf7!Vz$liTst6A)Bk!$NBcWG*my2R;bHtkBgv`iBm*CpjrzvHvv z+e$o3A&ElfzrY&$?Z6Dm%?p{CX$2s8HsG?`cC!Fq4lP*+tm2h=D?rj{cR6RW#glLz z0mJX#X&Xhr7S)p`_{5mA?oJe7$|d0iT&%VN1H)q7LduOP`1~v=xpA{qhOgEtg?t99 zl(u4b*?wbt^JW552AMSKy*4zAa!ghI?Hg)$X`Bd_x|Dd;lO&j?E!II<}o&_dfBd&u?ITEV7rYz36q)OhkOz4WmEmF_#g%_UH){6C(4SH1u5(f|LYei zU$OnKOm}xN{f*D~_(ikTulTBF@GC)3vGo<-ZOnXaX1*eNTP%DfByWs+#j-K(ZH#-r zE8|{`5xepQ?RwRz=u|39lEWV6Jf}xMY#6NUDGrE@fr{{g_{iQPWAoxv@04*x@Klk! zh2Ojt^79x3Z$(I)PCmCXU4-l3Qjcz(7_?ELzeW|hhyE9vlw!Bn09W1r`}FC4RsVbZ z=wPG&eTniF>wi_cyNd=`PK#I30n0CIT3`i^iXK>gyU_&OG{G{GTXew+syEtTv23)# zjW+nZ(gw>cp(ZYGbV66|pcBSCTqVrE2Wul`&&^b{^scoXJP~Hr1b|jE7A} z{%ad=pFMqAJ^y{Yf3Ug#_luOTSpRRJyStbG8a*R( zBS7_a%?wZn$E^E$6nBtRYy_20mm4cVo0XuJF1p!HP@jv9si0Ukrh<*B;CE#z=yYpu zR|WEnTqisv!zt`n%T-!Hp(+w-M9kIr45-Df#G|ZXM@LOcUzX<9-KJp^Vq~`lRX^x< z%{O_;1qqMf){l$qF0wH=L)*v4?n>ik-|K$v``|Zj+J`G!qR3@MS3G->W#TZ2lwq_I zD=MdLe%p{n-=Wqf`hfg0m7F1vC5S}fE}EnYQDyJc;as-4eA_#B0fq(Tz$r@^LafSC z=+s<R9 zLX<=}iX6Cw`{qR90#LNws-c@=+0SmK2~|m^aT<0JQGvhH76%aUI7v+w=rq<7MU~55 zuI7TLGlL5fSs>k(2vceDbcSJDOanq&25DQFktK~0&2P~&cMn+x5pW@hOgMvkVBpPh zK92uvOi^NrZfaZZl&Z0^LV%y{zzwubqjHG*V@3_*(I0(ljtToDmx!v?A_ zti}Kj37-P0EU`6MkzCoA4U!M@ zX=R9|wLfVOjdDeQ{ZQH~%eqE|RuOUwpdBQgm+C|nMqx#^Yzj5=%W@(cDRmcNbWN5f zhfsFac575mu|RX?-haa`u8FyLrdj^2m%fv2$~9>aDVj(#&vTVi()P z$0|dviJ2|+R>hW;OMdlnpT^fG5nawR0o=^^Luuwtubw$gY)*0l%cAcDn(`iQ7jR=R zSyjhu7oBjW|2zIjrQV~DIFTn@!A~E(QikN-bh3lItcg)Gg3pQ1Au|zW8ec`FD8H1p zc+nu_vb7lLOKWrDG83DQ+?6kkGY_`2#-0qib#Fz4-?`KF4qrif?l<;6Df4m9-U?m7 z-QXCkIZ-CuxC^g9GM?D8xkM0UuH=Ht8sY4wFR8)}9i~@fM2K)eLU+F(Xt!9A84XZO z#EhUsZ!`TG@5$dyS(BQ(LKvIN$HEQ$p5E;|+k)aTtskLOOILJexpru;G>0x3%d(mf&t0iNF?Cm_5<%=f4__JjbjeDXgz+k z5`)c-T&}DlH?LqNYpr6i*=BcNHIJ)kX>AH>vpe7aQHABnnLWEC#NMf@21R>@=MFEm;;9`X&k2QppDaX_` zxWu8^-g+3O=At)d4L}eBod0w!U0n$%<@CJm5CHYr`qq)KtW};1doECiyTLAR zPO?g*#HXFzs5vpG9mA8OuCXC|7I?`M9`V_-I32(=;1t64iIGevR)z*g)5`p9%Hk$87^WB)&H$p~h*$5f7Y=n%})i*-MM#yL=8zEyO zWPCm5^*-?50`l(3^Xf*s4tsOQ$k!WlM{{`yz$K4}D8{9gs};BPyk;-gf7tmOhOv(8 zf4^G;S~ieNYjy>jaSlC5Xb{V6Zm(75jb^`WiL{`mtlzmYDK<~loFFA=5)t$#Ez6mp zBQl+mK>dJH>CRrVgP4qc|H=LJLlcu*C7+od?&@HD46V3}2COJ)Ms7OLcmdFTbPN{P zm|!N&Bk;K{R_>5mew-7Q8}N7ar~D=hxdKu+;r1Ma?4mO`(aE2HxN^eHiAd||!Aj2F zA;$K_*lUtS-I<}6L|c^;mlbZ|Cdn?;U6W-8-t3-#ZF&VkU^gcrCmINiM3(A?G(~h(zq1Z{n$Nxxiv1MQQPL59ox2TYsa>ee{9>fZQHhO+s=-ion*&6d7pRY%uLnPsXA4wx>v8e zKXmtp?sfmJbzfb#rPT_Yn;Yp@>>NPSz7%LhLL3U_y9o#GGjZIwsh?o`CprdlQkc`cfT^N(+J3)8!X5I>=~>8feb# z^=e&7T=u0*2O$jcyzBTr4yWGs-NH9nAcsizNk~G*M7qJNKgwUUVDkX;zPQ#I2eKF= zWTk3}=4}U_W-jn+$W6!7uen_3Md`H+et%bWVh@CbS~4}8Ks}n}_z_u`bPP?^p1|GX zgXElEwc?aME2SX`#BldEtK-zm^Hh$~VBY?;{7tHcZbCyr_0mxbk(-5ca}67aggB@) z?68{#5~ zGZwzP)ky*7?_ZV-BZEv#9t?y#DD3^&(c&fZ=dR@So_Uw6o3vDj4!W|zO96W7&)w$u z9Bkx(`mzX~Yovb{n&#Du{D%L&{HcRn(j+FrM^#)l{UarX-!^)gv~t-=N2t;3(qObY zY!Q}}15=0t@e04)-tGKej;=q}o}T32cM1HyhMxAkUkt$eKV*CJzT$il~Y`E$J^YJ@@*x)hMY) z2`sWlA;)-4^r`9=)15U<+7{}G#6|e1^BmW$o`7e86DeOvP6fP1b@xqlJU1#+z1-ML zTy`4Ng$0MlP6#n#&u{g~K zJzi&!iUAHLVJG{IONa7C ziT_zb9HiAUPZe*ZJ~?5jJ0wd1UsC^6#cvdDIc!paJfZq;nd-#{OINm=Hw>iihzu$q zQsJRNG8Z9tR}w|V9t_?WvrP?w?KMc5 zA?5us583^OrarE`q8DXu9EesUvD~;T78$mN5gU=_=M4f%@s<4SN zM{$+ZFdA(zATgXpOk8|RDp?@sD=hw2 z4dNGjs{53~&oU#OoGLDZc;uX5Mw)0(jcIFq$!Z|?AwhTZCP|ZXWvVPCF^cpMKDk6R zciId_GnJVoC=G_a&DE!~V#0+wyQ1y&B<_LD^(5`r=T$R~r{WdLpOJj))xFW&<_3z6 z#s&(TD@nbY-&8;LBbpEVhrHZt*mDkS9G)cDvI>zZ=BbPGF?{>oH^lQGqu8(*&{61h z)7YrW)Hg7&w{C?4JQ5o7=O!r$+e`RQ=%wKr`Vwu{jzmBm5=*{^&v(t?0L~sZ5s|;7KDE4V1TA zU{1Zh_96BolqlQNlB zJN&L;6=1}6FX`>pGTmcfa|9i&>qXazxBNrQV5MFQVp44}W943QySJvG6m-LPjr<#y z-2B>_>hsqXDqT|){~MnLL8h3N9QdqC<2>gz0hj3f?^hdWC z`U8!HfO$@NyDB(_Gf&afMG2t}_zOEo}Z!R&cXVTb~^J{aJ zZ^4*u*tVX}E!@+sL*YTTX5)fZI8F-utLt;+L7tedmL!+j#d%x(K{uvHf<8y9y4fCV zE!E73`R^Ra`C3@ep0ORul67#VI-)%oh*d(XTTD%XAc(DJ1;D)x!_w=4(6fXtv%oNs z6bVH}$L24_yh3C>ohHi}op1(2D{=9P^8bF9g%mvV_w$)XJ-6fbOkzU>TZcTk6sx`` zfLzf-tXt)L)x;x3dbe3<(8$r>DjjNWMvPiVAPZ&y2g5Rm9Wp_mX#5dCA~GnGMfH67 zFi@}RS(h}!-S6ad|65w$Z_>Bn%g_65&+P5h4I;$(ny!zY_;e<%bDs?qK?pOcj767# z!k?)gK=!RIdKP&rBG&I(I&?x#lm$C`&OC0rvur5dtU;!TrE7__|EGvnGDUj)uy9ey z^dmut0Pjxa`EiEBpH^^?RU?2V33KA8L zZ7D201M2c5(%38qbH zz(%N*uOfAIvF<%xSV2)^cT>|1@i$8-+A~8Ju2(5v7p>@PmN$^IU?DAAM^EqyIb7bK zKX2es3Ypn3k#goDD~+G`%Z?5JMHzahw5KS^kN}@{9ATm+)^;*OH^I^ZpWAiq_OgIyYtQ`O3csK5`~99}gXF|Dy*#x2 zO$-2rqaO227~xNh(?m_|O_(NKQ_uKE!T-XYN#U7F>;~)k&Ieb=fn2+~wL1GfHPh0b zG%mQh=sZ9uczH#bWs&KVr|FNM*m|y0{Q`lzqiP$&$irUc5xyVZUh4RHKVB0S#gK;D6@L^U8(A7lgZ?$lz2wxxqtlh7-z9-%3PpKb}O<##+} zFHXCzRAawkyZ;D`+TU2u0Kf4_|I8R^KCSj4N<~W0g4F+=J~~xx!ea#HFe6ny|9;`Y z>6Vf5U^Zr5pjBH8XvE=JGQ5#WdOb}A@`9F^Q&{$DRefO2&M(j{h8b@&xk%?}!idMH za^H$j(I}6t=7{l`lwoMiF*25~Je<>49*^qqgMj-f>tZ`}95T3B{>4pL@8f8JoXuLJkx?OuKdH3U>FKkW3GAJ%VS{jg zmD!Iko)1Ht&75AG6h7&Zr%Y`uSINA^oucorf;+p@pAXrkHCp-TPHq2iQ7rpTrwVJ*43ebmPFNiSPpyr!PptrNE|^-gEB|!2+Abq*m&iLO)kU|L~`c!ebTpeM%*{PxU2OSKRFY|)D?x~2`M?Sc&>`}AFPd;OpZL0 zzUeSBpbI@5Wxsf6LChCI%$a-N;|^HKHqBs!CKOVm;HmMQBMLB~F~=RD7$P$5%dkX&7^!5U-sC>!56Tkc z7A&eLsjV+;?$@{O+`=f zW@t?Zy0>D)!=@U?3{K+9+m~p4oGaUa++-nh*r)pt$eU_wbNL|>xVnZ1-@4D4+NPRB zuL(bfW}lcooyGWd7Uk>C%Gw^5tU8zIN1r!gq`IMtb6cZAwC@yTikC_EIH&AKgmBA7 zd+Rv3dWJv2lU=0S0FhtuA}8c5rq_vYf3Zs088F16x2vCokS&T61Pd#2A@M^nNG-bn z+YgID+B~ZHUybXKZ&TJ?%zNq`AtSN(SDzAg^;DR_F$)+sRE9e3xbiwAb9p#cjr(vL zr=2UbBUqTAPBbhglmJjd;u99+J0R17Kef7ZtG)N+O?)^V*Ii>OKNE;09J80ZMA3IK zEmKmnzv7I%S^sLy5m&+$x3NVUmZu9r<`}GKI~wQPE*NcF>OL_2yoJ|gW)K#d#;#i^ zom${M858pApQn1C$MV|z?yvOrI>mLs1Mstl;Vn@OsMKWvdoB)&7ERpZoQF>VwT5gU zw@k|yd5ezirnd}!B0p9J^U_oJj_YF<%qW~!#<(%>mijkDx7wk@%(xl+HCOTGen$Vi z#&Qa!@cUkM3`1yle0ElAfG>HlF36jGQ8f1y+4noL=HFz^eLN_$<~=F1S}rI56Vapl z88%hmD0qx1-w0mavz+eRn}4fC^@8(Q$F^*#^x1j6)5R`3C7}lCC6oT5aixVo?;lsw2bhvRPx5M{cWeTQE9iE$r-o7Y@LS*Yb|>uRY^^JL)Xj zj}->?%R24<4*bMyHh#t}zG-F#zqRJpzp^G0cS|zv;$h2_C(AWU5_3>12M0(Mh)nDq zaJT)$tUa4yWK&lSnhm@eO*G7*c$rxiw} z@6t-A0)vfXAY)AUIB{y(k4Z)kqk10|e0xf&KtrM9rYYY-&=`Evkx?O3jw3mYFM~hv z>5g@GdIfXAw9~8>92}-UBO@(k+#h`KSwbcD)%S!bl{ubASzO&GKotioC#l)ETQ6eg z-L!`pIK-Y3QO0O|Zsd=L-o#Grwk9HNaFEBZ;B@oAK}8iAM1zxd+HKos1&;`ot_|$U zM43a@E8;?6zcSfqduD`CMY&YSdmpMHEiidAa17E$0q!yT!}3@FWT1W3M~6v#>v3xO zO)xeqP^jUy>j-f-*X-C`9$s%)XN65ilKf8g?1}BkgC>|q9qXanjQ2W`6MRmYx14SW zmAc6AMtuM^Y!y?#)vK(uf9krXEy{w4mNHS{MM1F3!JYA#xbX0zHNeef+I_XoMgoMy zV|NuV8bh*m<$3ixG!$1R9tz`spyI%1N%j^p>T^^89Q5Cz(|oBYE4*=d7N~OG;l<6o zYS-JJN{8fe;}d|3cJrd^E|7U0PG8|UK4y(apSGYs{svYcqz%zo) z?Isv&!=Tlfsp@%oH7HywJg${@u_aIuGND{>jS+>7oDFhwJ=kIrcfLf-R^WE7UdpOd zHi7xgvpHtLEA8%-HpTpAj&dkjdx`yxXFLI=-C-Kc9N>&l1x#C_6x!$Cu|#A?^xOv` zx%OcEJ`Yk@iC#-A=vK1PnMmGp-tZZUB7b`Ak}{SGXG#zqFc+%qr<=o#85h%52~SIH z7hI1FBQVW!VGydGowH5;&=P;l=Cr!KKnE7?re{+;9p?6+b*3MJO`uf&80VDyc;1yI zQxPK@;1UUzwU~DF7DFTXl&V*r^GDdnn}H8_5;qXDVCScYV;OHjp8UUb_&qx#ReB=6 z%&qIoq0Ky|ztOkABxlTfrBMu5)312@7apO7fMukf>NwE3FNyIFmK5rViEsE16RvFh zubo(nw~mmQytyrgtDQA{&kP?wHjVnqEQnG(&+fV1?Sjxy$PO68fWNVP8mK8tAbj_a z^h`g=fv;x&_VuzL1?!r(P{)@(U1MCM3d_zvcYaJqSw3^){Vk4`N@4bNm590JSPPWJ zdI&utLbM&)QYe@_%I?mwZ@~3l{2LYu$!yiW#-b|5)fUV~px)WT;V=}&OLhQS65k4v zN;$T|%05DvWt04C+6`d9*o2|>H$7pZM1|Cr2V_C3XS|V~`qI9gP^N@4GBupmTtU)I zI_0W=#PY7KQ|x_g1|T1;rV4>P8kM0%WB{n*#dy-rhj$Co z+08aik%(vuS>I}yiQu}%%*Cy71T5I--3N<%x6;amNxmd`B>2~+$W_C_aWPW=Yh$W= z26HZYMV}P=*J4uiT3g$@;LVX~C;UT%ac10%boU^!YD`Fs*MA)N@Eu>@n-kFR9XN4D72Yz zmw>ACG^DtVkYnZ$+3#?MJ8G>S|tXR8q>_y>SPR%%7*HhyWvVNQ; zQ=-U$kYKMm-g3$) zeqOjDxwgc8S(D`VU2h|%pD9!1_g*NkpE$7}?9c08668*^bzYGCV+sTzq!BO3#=LHl z`l&A%JL#+3`4;jwJrN>djS(VxG2ODHp0TZ7(IThH$oNp5gmCB^E5lxI_b=n#HrIbW zVfv9iLxL$;=h!OHn0)~(>i!u`l33C8P~T^K4WsYt7B%SwOHry9#3UD?QC559;rZ;F zUii-k;XV%!0zH^Cg8ZvkFVH;+kS4Ye=S(XI@;fe4=E+hcM$WG%h2>1@wjLG$O>7(S zvZ8`A#Vf?5Yr&bL&7p`2KdCR`DYhWLs3f$foRsW*m&48SldAroP3?zQ6*$g{HyboZ zC0Fvy+gAAr&dPR5oi3Z+1ER;ShHQL27+Q9KRBwNvCEE^l$i`<+#>+Z+t4i=>muV{3 zPa3RY-oW>-S3ky|SgXLLky02lrgQ&-;qP5QtkFQTf4#DU1TVJtcNVTcO>!QHzc2m}16!mWU^mYH*H-33q*KU^Q zBTo;Zv2~CwY5GkXNJ}Yi((Js}*Q(G)cIhoVR^5dVVi`k`(wmzJ6!3%eAH zUiD(KUu*0XZK<+BsyZuA-USj3<}__e`mlf`@1nQo4Kj{QDieg(&io}cs&j;MN(p?B z&Fax*?2i&Qdtg0Iru=efpuWqX+}ca!-FmE4?EshAK$o!~5^wZV20ya7Lq}k>{>Skn zT34nYd7;}ZQF)3U**H`b6F?$=r)0n(nDaYf8Ylb?Emuu^nump|#)a!s zp>^2gb-=;V&5=9<_c!Kv?_cP%_4TIou2+FnUBe97-xA zyZCH@@7twF%OS#f>SmvUdZ`lrQ>B=>#ylv5sy?+&`?{v2Kf%3vE*P@9opPSWj1)|v z{6wCIuvH#)V8UdbNSgo52-Sri81imXRlaIQ2B6QUl3UzImIFGT<{!mvw?iKqbyg-v zd&F|bauDW5Z=$br!#45&GG4Qr7=~C#9U(xgK^EQ9nyoBqh-=ls&^>jzVNR!{mHa%W zFguMRPmB1hLzBnwjJ=RFe<$C!k@e;GC!iud=UO9mLs3|lqfoVpt?`m3T?>w1m5_le z>J^&nyba!yGjy$a#>RCOas0rugmh&zVOgUSxmM%nn9my=G~Jr|DJ}kfhZ#|jjN|dX z^#V=yu+%();?kH8k-g+aXlH^_$kM2pS|)}yhNGz0$v6M%AB$LdBK#0-OF%gThAD$3 zKPN48+&%S_GQOJ60D(@?*HFQwJ^v#B|9|F_W^5BSnP$A+lujt|4-jscFo^xX;l(bA zgzOUlKA?&f`X0+OSZ!^IbuC4YNq+mI4}=tVERk~o<1jeC{HY}cwD20ht^!E8!n&U#a`moH6BEYdLLp3i`9Xr{NKi5L6k69{Uz9TR>fu-qNepq^Q6w=t5PRm<{7Yz4qgw z5(CYSNh`uF0jN(gx>r1ErEqe@gs%}mTw46lVrNy!s@%{jq3dxrN0972o<-tpqt*7G zFN>FIoNa-4MLjVfY)t4D9imy*dW*cZ#z$1#Sr9YfUglPkfSU8r;OVEZBYHzwk=?5F zZl!P!b%_$~$dNug`(JsGavG3_6tNB&7PXi|UHOeJTg3; z`OilkDE#=w1=d_OaNKj&U-!Re=Pgp`0`uOvd49odYTzDGstu zMgs5T;BubXXo`#L-utyRX5ai?ew#a^F|@Hm|1$G{5puj}Un!K9Pek2JZwjb3_gh*e z8WTJ439zofB}H~(Xd*cBhq4fz(7;7I?XM6HBwxxz?QsO=9U zBy}#sXWQtgZ(lh2n1G824LB=TyOEwS(Xwl(sQkS;URe7IC&hn*b~q_R$5e`b_t#y2 zdLE4%Hdq$K_Uv`BH5_f`zM6fpasC)!QKV2b!9=Ao175e9n1z3g)_T|>-HM6;ei$V1 zJTHUV2#9Mp47Q|~l4k8W^dlncNn&sMoO(O)xH70rUV1skF6*?9aqhcm}fTl z>quc7{|fVaZ$&hDd%E|Y$)qwO^NXFDq?BJ5#0obr# z;z4T;wdGuEdHN+PxUIy3S(KJde6#jr+~$geb~$gI5iqneb~?b+ej*WvVZQum&iOnP8SVrpA9BKC0Fo?PB%eUCMnpm7;tC zc#_VXd8e#T)LYGK!}q&N^Fd(K7qpK@@h=>D%~NkH6^Uf#EuX^xX@LBb!o=qCw9)j7 zF(8{N@340`Wrw(rMIJFrvFye)izoiQq4N?uR*Zj!b_QE}Wb@&W>#^K~NxI=z3U04v zat_2FUb*|<6p-UG+elg5>5xbVx&EhY=&Z+Y?UjzA7{9eK4j@@)wt@LTwW0^XXqSwG zRxBEs`Eu%w1ZhGZk6*i`SE!+c<`O_fT#L9=r4^J9u0x4GslR(wpnNRz)^369~ob zO?dks_ZtuP8;2K^y0T~Hay=Wku~jxUf+TX2^OGPJ+T53yQD104-5+JLbZ5)Qrb*aO z!Fk#SEfTCX4yFd~qMhYCVXv3w^I;o24V5Vh}O3GTfI< z5&K*6Pt#jR*%OjKE?_8P*+W>>XTC=kB=cT?tV;7xBXr5YCa#_ONKTuBX53`h|LSjb zj9NA&uy&1L9m%2LiKuW+LlOQ~`7{x?2yi)D;jUjTP=EXvG+(%|Mbk{Ck}D?*hle*nVA9#MzK>3({{!Xt&1< zER@UixO%#>@kT58=(EE+mNuP=&7x=KlWu@8I17LwLAoG_fr}Tdbkl!96Z2FWl(#7~ zwMW+bW>}Eyr{hV+*84HFC2jW(23Z#d%9Ib#!TU0@f`^vF(lnaWj$+g-QqC(z6vi+k zC4zy=mJ?GT-l)6IOoNJuUak^R6rctBn%Q%62YVTW6SHyRSrq6d5|#Lj2V9oz(^>P| zPV$omQI`JV6cW{_+&r9BnRpi$;b%XZyKA+ngp4#3Rn>7?)&;sF(C=c(J%KTS zOyd?^q<$$|TE~97LOVj~cJH-Iym6^U1Pgp`!;N_qY4F?^F%u%k?{)a8&P+eu9?(}Q zArQbQOMArJH?(KuTg{&a@J{H)=y!LN95BG&oxbti&ns@YdzVM-Bk2qF@izI&A7`)f zWf#_)D$NVB6;Fi5%$p$Ens34x2_u%@kQVtc6EW3}Nxrr@kR5rt!L0jQIEp8p))h-h z@nLHz@hJ~vaAVpPRO4`_02@IPLd)NRQf8&0!1n65cax5Kq%E~2y)DZaYno26D|4C` zTGBov!T@QX+v?|9-&LhO&5J2MtBN|z<(6f2PByCwyAr1FloCFiT1lKJpJgT%d>Vit zB968WG;NFBte~?Vj@hF1<67dmCbN!Ho7IFfT}Cb0(oxfH*Bd~()M|qDy_#&vP+=|6 zVt@IkmooaXmLgLnlcQq_s%!pT2sfhM=rYS*xG4U*iLHZi_JNF5oYww@ulDw3Y0g8y zBh->aG1Q{lXJxk~RlmC1`ZE*gSZry#-rQ=v1$b?Jr9ACzZn<9DbZNf-obnUUTiJS| zc}+vUzuPX9OY#8;CnLizyeFWzKc=BXO<^_nAna^OIenwAIRIQ9uDMxUb~QgJth!nJ z_wE1Z_^PQ_ng-m(Cryo$?I-+Sm)s-mwJsoxkMZw46s&ie9a$8#w;MA@=bLTtdzbH7 z^@Csy`kKkD)NRQoy*Ji#Zyh4)j^dlUjpTA`;*CmsPj0Yj*YBtQcwOwf z+LrX69x30YM*mMM&NKxZG?cK1vFFS2r=_P7JdL%M9gF*2IE_AAvYkzRjh1gQ@TYt% z8OSd24w%|XYfbxnv1T1~t(LKx4$5>}9ta_s9I~8g$U4OmnDq3DHe4hT$SqwK#7Hf% zR7(iYeY*wo^_@=3dO)#bgdD{b^jpQlAi)438cU2!t^L*QbB#aBzE*B>2p=1&9PMKU zQB(2=FL=Lyya9xZ0mw5=AW%C+A4XS@eZY7h8voBJmJ_l}M5=6jQ@{0i*@056hj`Vl|Min7}j zI@&*WA!diD9-+Yf8qAQZd4%!wgnjWFh%Y~qK%7DRs$%m>>7LN}s$7S$V85!)x*T~$ zOR2u?VzCEqrN-A_sd;>9{7oSxmc!aort!E`b~jl%Td{s3*L_r*gnc`rmez5HdNAc% z?z6-qP-VEhDXmd8eJWp=*KEz{06NDpx}|)4=4%Pya0WyM%6a@E;+#5iUM97EMfnKn z9)Fwy(ci>vrz-AIj_a`?Y{Uo5cEr3{oCPzyw*LA;zi*^HdkSV@^~{(k5Dz}2p0JZf z1yzI@V){CmQ-`w($$1QvojWGk(}!`w4dAK1ioAm|aylG14o79|au;=nMV9)*Z>`C| z$UD}E$r_}Ndowsih>;h_7%(yc^cZ8D;r5B@cYu0P)M(3Z>W}b>SrK>ZS>8qi=yDQ3 zseY66SDc4ZJk4C&RXr8lOST1f{=a^3M50~NWs2X8zh;nZx2BI+^;Zp*^X9(>5z=)+ zVKA3}p(4sZ)7+IviT=vp!H}#qA+b<`yEbhQ#Jfq9TT|Q*zj5R*q(I_YtT-5>C0;_R z{h(Hik?O^JV!la1dsYVdF6*9FRGL6Ar;ir5^orxe zYgLptV{95t1+E4XL`!xRKAx)7A0;wLz6}VWpg+IN0Je!};i?89Cpf>n&}LMrhIJYj zQem3R)ABHF#6dAMLYwb^68)IwP;|RwbWP>V)2zDkryOgCh6fRDaqR6i^lDJnFNfD&SJI53#OL>lwF4CjGx|M{-o1GyKWA3KwDRc8js>SIpBk#?>+B$LBwh_|*=d-&#qEced+ ziku|wF=neWTnf=fz98fW46*&p-I$C4{G?u8KjvY2fMRvn8LJaY_}5;RR68b`S{EC1 zg(UE1o|q{S?i9@Ln{ov8W}^$1q+r_>)ZRqv&O?Z0rgilFqyst9IApT@&5a#mqZTQh zZQ-4C_P+O*UM)t{nbjHZ4=3!OZSzdbr}LUI^$3_P#I9J zX)?7?q7~DK+%j+U$Xw9lc`!*~L@b7!C3{7{B-D@JgX~L+@!pZ+dI4zkMSCX*{93=B zZ1^^@c_QbG-AnsysvzYONq0H7?Sm)u@3{VT!B}cZKQ3H8ddfdP@4MBlNq_Hdc(ner zezxO*=z8~p=$doQ!~3Xd>+15dzvWnbw=R<7Eg(?%YuDrToPxLQcJIucgYMG7e8x$- ze0*ipkNcXp*W&|kYV$8=mCkKim>Kd@U|n+$$kSyx+Ok1}m@j>WOqP)}?FwqjP}giS zyOy-zFkXxNU%tc6WB1z5&bibqK58jxl$&I+wPdS$E8nBY8Ghn2hY5X14#}UgZB8_D zY06r%N8hAx*J+>ayIM5QyjHQfnW;)8U@!PJD(RYr`b~YB2n&=gP}`q8N2|eVCEGuQWc?46PL;NUoAT- zQJ-h-b8KN?Cp^oJW~d#ed+AJBmQs}3Cl7PtYhGy;YXC5JN2=ycNGlOy+Y6RUyBjYM7mvQ7|Md3yvzH?JZ=rcUwZaFCZuG6VvWl?}$@Uh_bo<0Ut04 z%iw)^Ew|QzSw?|RBh05y7o_%P&*>#dNsb%oPoQ!~JSk{Q4Jzu-aUTclK8Ds`&+iWy z;;=IkgazDw*d)zlv|RtQ2=g3`hM0yFinCEpbJHh;zxt@P;fstN3IySBGacg7G4wdv z75z*($W};=RuhET#u%Ea;%o-3Mx~b$@>6c*b7b#eG0+rv3Wl>`9&-l6R+Bsl2N^<* z9Y_RZ&0uJcz|a42pO!ewWF6O1i9^1hz|f_c$8tU*X4R}A?U_TL)tjf*)M;#%7` z<~`A*zEuc}qef0|P_>(l;Vr1k_MAx9uH@?*2F@mbi>jsyh`&sGD^H7m74U-LR^f1J z2svOjLPs&h_wb%ogU=w>zKdoPY(D z#C;spxbo8K2^XPSX=XEr``Me|6rs_4i22xBVw-?Pu6N;BQ~c_wA;i2LE3BH>1-DoJOvZ ziSc+g*K;t*TIh7e*upb1uLa1pC{9D1iM4cS1+8x5N%q`)hTc;1Qk*g*r6QOVMifu* z1!42?iLESWUP)|BW=*NaAvGt+swWug`@#VfoKk*WJdN8l0;$K>uuEvo$cbKP zZ&8ig6v~yRITqo8dJHpAyjTqsm184iICa$BLc~*2QoGs%uSXi_$yd~mk z3@T@Q^Ijsjv5vskf8stJ3!BY}EP_e9VBB&u4RW1}RnxFnQ0I5f#L6v(oB%&Q@@|db z>=QzSrA_dr=pnU92W(M;dq1ybf@`C*%akw_4olJta1GC&zpSwhCQb_=8+mm&^7@TOAW`hF^B8NX>HeUX)s(0GBH>EMW25Lg++yWA>_s$5~tDyk~ZjC z((SN+)C_G~9+_KGxXBJBB$K$$v1tNsR1##4sF-Rz%g};{1IJ-vK=~?VUv%VTN=Fhb2%oYZakTmhIpx z+M`;nHHxPxguvf7)^SWTo=08mFWrV+ z5ac0N#ZqZ)V^C2*eTO0d<7ug}xp_b>Zi4PcqvcS?03W#o>G_a{A;M1U+>fYx7X1~m zXPBnKE$;iv^rg;^&-a_?mCj)#ol*g&7*(PxmT`mX0pXikt;P;hDOinQzTAlXL*E(y z^AeinUC##b%tS7cnJ40)jG8c-IVRq(H?hh=E(lNrH{|1ZDmg(A?E-SLk$7)YBJ%a* zT^9O2Xhx;Uw4hJ_+ZiAkhoi{At%_#1ZGMzc3+x@8oH7A|(8Se{&Z^zQAi_-hQ#ZWo zZ{8iLsYa8kzlItH-LqCV6ap_nR0WsQVa zwr#ZEXe(M^&Qc zSW-ZdB8ACfof2}sdDgr|F3dj%V!D()n;j!se;@=-OsX*V+*{(BxUsJ`%wok*Ge>;iK4-8c25QmhT(=QaVF$Z7Lu-SsQ~| zBHzqzqt+*iRYkn6t%WBInhykOGQZ;MVk*DZw=^YF5vJxulbRRFy>WL4sj9qVZZKEj z5EtykHj0KomapBb-;N{14$P2oLtdfgCT? zD)cP`8-=5=?4n~$xRpgDKH#va^q=NZjTmmWeDt=@8}l2e*T~;Mh+Uje$OJ{_97*HB zV8*@~)NqNXWe-ig z4uHM(`0sTb4A_!91w7Wwsz;ZfRLooUurA^j*7b&|!>dCTTo`0Vm4*6|dP!&SHCSE& zf8rItJ>tih6Q5+4a#3+{!8~!T*wF7dMo)_XGnNA z&SAoHC}iN~E?p)$h@iRO^TCAZc`RYCUmX;qvh{zo(-rY1)(SRR{>(bBu)d>A*328J zv&6l!7^o^-pb}C{#&00T&P5|GJ5*BfGiwL|Ne5LSA3Y*3)EmQ0lR6_a#!a7T)5j8E zgBz)a2Fi-4Lv{<)keDur$_m_Hjlx_r{p%l79KjR6T@(vtxhN`axQE1RiZGTv==gbS zAU9Te%8rH&%%$OM#2v(jOZMY74&eRnl|`BVS@>(&SbefkS!`?>Kusx1<@ATi^4`QkRK-G;cDach-O5n$?IR>Sgyv>H>PqQQ z?Y`Hcx3Pu8@$27n?hKrS!Wq3&ZaMYklF0ANC1an=WIn%)B~ty|n#8D{r}BZ>OLg8w z=&Q#_BY6&G7;V+Z?0Ip)c?L)poMXSpyGmFN(2gSBd7i`6u}S}XVc6)0;H2!t!IR4( zr?ugeTH*Nl98VPpPfLLwC*eW^Lhj0xcO}P-RW0jE%Q&{iax34U#W=5#c4lSi=nByf z9(+s|=MZ?10HV=Hi4;0R(I(!im}XjM54BCoW%J5yJvN+fO=(-tWcBsfQ~C`hUd-q@ zB=RBC<~t46gGi*x4obR(tHF6=bQnxyF5UyS6G&+dWqP+D*JNPgge9d~LsWXkKXT)c zSu7C$-%1NsS+9S|K3^?7%aqc;KT8jCU^N9evo1A00;3E#wA9K!S_{Uo=9AUKi(*%J zq>Fj$bpQUv;lDqa+D^q|a6NRtZ%d6Fj-k}h`KwWBayis9O0Kns>{rPC z-?bJwFXC*Oyf5@Sz_rQyXPxB&h{ewr=|-f7L!)`~eX}&0)MD^`7%T~tZ$p^ON|_~& z=`hK}q0~F}9a-Hq>SLBS;GEoQD|minMQ6$% zlv-q%aqXJ;mrGE<^dNS98Bp%-&HX|2zjT%q@MR?POsI_{)hNYwf0In>dTv>@(J~ZQ zTJw_nUfw*n3QaP>GpJ+mgl49c-#WJ2l%^glz3R2>5^{h${^P|07 z_G^aeuFP>B6?A0Kg3$}p)WSP9)01m%$}}r=m%I^A^Ff1UYgMr)`>o}gvZYP=fwZU z)w%%FV`hSGX-2*{>J!BZX9PCEL>e@6Orb|@(H)=&NMOt9`8VZXgaBPL!J`Xs?kY$9 zX{l}9qbmq}H#)H|-zjvhbXU9~@uJLM#?<+~e1L7bxs($^f(}QE6%lwYG!%H?NUgU{?46bw&2XveA)>_GAhqU90G9 zfXtX(zwLnvEI%^|tdx<@>f&Oggr2O~6`Yntcf=`_W3dEnfI5@oh-6q**-a`P^@%JP zU+;+SfW9A%7!lsMIi!K+7Y@=k;*$mzRt5bdwosktC?3jJ%|)*(9~a4 z=kedE?`!Nou-pvNgjS0tANbc0#brUl*4b1NtV8=kmN$vV2GC^&F7vY{JwX3={ERrJ z9nWK%0sg4+Aw&3U7$Y15)jUqo0KX1OJ{pZrY}}~=!=9wMyeuhHeK?p0emR`jPl%#9 zV+=0Z`@}(ylW*hJ8TC!&&nwM_l?*n7B3uva*U6*?vRx#$RbT0uV~P!91c$yC6uncz zr>?1)ZN75RCT~ekctmLc(*A~&6n6!u^CX)ki$k~wX`!?(J!_g4z4)DgILR|0UpeFa zv+9md<8}jvLtTIaziMQZ$)J!VnG8x&?s+o=nZymX;q|)NZgvJW ze0qAkvTZi>bzFvJakkfh(L#5!%`#K?r!ipTBs&YX@mZqRYjz{&#BZs-X8bpKmvX}| zpPe7f>?;`3gHCVkmP`o#)~U!QWU zwq4SIV)=i8&IB0$6FNU6_z9+lxMn0zW!JN!7(v(U?`+p5xpwhX5WC8Ac)adU+(r=JfCLT!(k?>7$h zAv{QT=zKw0VLkVvk-g_HRZw^Y7iLVU5c`^MEg$w>q8QHdP7HPek!3Jhf;I8h&y+i` z8fdN@h=WQpZwwj)O9e28(_1qh__8s9YtHi%h_2OCjqjpA2GPWE^Yy!p;3QT+PiEUG zI?>-tPmhc$6Nr>#Oy-lCfg(6B4}!ca&Mpby9;cf{c()6W8RxddYhXU3mz;^Jg^P7x z)R5fZrf5*8xdL@7MmcsSD5}l$!!yF3fU8avurXLH`nynj3s_nF1IfvK@{yBQK{+R@ z0-$Bl6B4$=lj+Pb{1<$_Wi5mwkHVydkk$d5d9-m=gYcF9K;rPI97#x?V0~ z7o*uHcLL_ypO&^I=`Qz=Y`BZhycz2yj)P-g8r`J z`?K(L@c#gsKxMyL{k?6>s2BJdYx3ypKz{8msBJCLN3YlGJ>B1j-@RTh`@8qFxA(W5 zgWdhTgWYGl&kp|9+uhxNw*NQM`;6m`p3o1d_qX1?`^paPJ9!d0-^qNpMO`=gcc;7C z?X}WjZ7w>avumQzBEB<;OlK4;9l2a2i~Dgp^!U)wq_i~?HalF=*+Ik6x@C(y;hFwn ztAp+&E!k5vqR0RI2ifWF?RI;e-s2W#M=WV6qFbH&!lX`M*R??G5l%~60+C*Kr`!9w zgmB6Ik221Z1fY8V@9gg!!rd!&dD^HXenHl8gr-kBEVB;n~N)qlKBc ziDuC4i+_S?t?Ag9UIStxYeqLz~ctodVXAxjf5YXaI6Iw zV>}{f4mop~b2;OaU<=yUL2W_BAo)Vn4F{1!T35Kt@nr=$d-EOAKObF6M$Lozt|xRh z`1gOi)fAN#upPq*~z-Git4_1^P+cs(2({W!e2 zxa{BjJi7X^pfavL0&xj2M41gAMWNLtoQmq^fl|lA)$!%dZd*#h$v1o0l+B&!6p31| zoR!dXbd@;o5F>>eEKyx8)Y;qD$h^>*N|z32O$O8W|Jp2tSc z3KTkl08`5PO`>~0^@b1^>UhjNfjdn08)o_R0F+QDTq71qb2x(~o ztSyEgU5^3I=cLHx$VvRfJARaxnejwAB=6eo_g@L_LNc`)_!3pl=FIcB$<*>rLdz<6 ztwk$|5B%dHX(#u-EMPeNC3ntgFgqmeZQ%ynhQqhV+<}kCbzK2((vFAI2HF5xWojk- zR^ZZSgFEt57%+b`O&Y<_C>b5iW44&14Ejh4eHKH{TxETIDrqtb)EJG9Z0JAWg1{8a zcOEO_T#S1NYIV3$r_2`4CbZn{wd1goNX$)U(ZyD5sGxNRgNw)<3V>s#wt8~d$AEau z5RSpo9FZ6c!(wE^rVA3#TSh{#PCST4-y%4uXdDBJQ73dEc>M_$IwbAAo}5ob?S~&B zQ@FCHlz$(R_KTgJp4!~Z3-|;rbt7pRJQyGgIBBb9I!v|Gku|Wc>j|3$GZy-18KX^g zX`GH-DP?zVu)bl)TAHg}oJ_gKl zsHH+zs_kJhZwnKNBf%qKz%Bkecc$7j-=^FNm_r>SMlC#Rp}8XuGsm|I2NoT*gQz>d zq$tr6Njlhk-r!X&($oF@Jw0sh{F(_qnJZMik_qaP;AHfl9mOXk7B|%^X=|1|_H@!8 zQ+4FuCm@C;Kz)R!+G;|!h)r)3&J_u;l>`S{M85Cf|A`ytLd44VkZM%2E$WsIjIC&@ zZEd9B0OH|h#r+SPG+_p$9D5R-Y9};07V{Vw-Pa|vCYOBbWWjwzMiVs3jBVE)5v8v7 ziJ|Q#9W(MKis%rY>ZzV#gwfqnATY&dEcRljiuQKNNL`0Y8l^59Y1iH+tEhj$ZbU`fpxNr@zf^ ze?2?@>8GRquPs0Df_&BeqS-ySJ`gA{y6)=p97 zF|)*-;;LJkOGz>)z)VKbj)5Ub>$*^s?jW1l($P?k;NR0CU4kX1#A0)1L1SYP%wonE zPL9gvEipZol@G}-j+6Ufb(Bt30rpl}zaU?rYIQ4>?^$1!fqRM1ovCW3l{j?>0JTsF zV15mKc|1dh(Ve7%0NAbM%vFpdqx!Ig$-0Db=%bBAP_jl31G62?tEzG$q+k)0FAa4ki-7fgf5D=Yxc%!5>9ZNM%dq5=x6`J?Lr(peh&0)XOxN6{Pdcy$Q$c zVoZ&5PkdY)lHWqQkZq4W>KMO6P>PN>rbY+!w}SB@7EXp-{Q06{a?Ur7YM`l>DkrrvKxF1>iej>|KIOD%iRC%@Ai7@`@b*o zyka*@? z6kp}fKgpB@kUyo>dIdLRI>r^)Lwlit{xZUWv_tAJSKs|6Z;5{Ua+4t(up_-fC} z`~POfESGtHH*o7r0c!UD-qZc3nf-sT)7x3^|1a_U@yGU)*6^dNJ;aiFRzQVr>&f7qNH@2bg$8tBzd<*K1oT|G04+B_>sokp)TIPc3VF)Y#Yi4vIjXnpNyG7C5`cgoeBvH zGXduGsJVuCnJSpfjQeCmT^}8tUMfBbSg*})YcwGqQa)@Xa*BWn#dkc;vM1P!-emeMKWgU(7|P*4R$S~D;An=h#nizO0iAleB&4p# zonS)R|K)f7%Wr4e!cLo`Uh$tPgiNfo+*!)@K&~>Zwn7~&1qryCuQ*hSmpeN$w9tfc zslRnNsZYdXt|d?5_Wx^>_Dy^F-6m{X=PUCY1B}HH;u*8-u0O+5=1?*T+%c@stQ~F_ z0je#oHG9XZ({2LG)O_?K8%Oz#x^Ct=KQhmgEKIJ-C-Pe;0+yMdd@Je(tutO6rg))C zynHR3-zWv$k~5qgbf*v^w0CSUhe%Jl#pBXmYlk@wuWCQ!ukdgdDBeNq=w{6aoV>w4 zEu(+wVqLt>Bbftgqi#B465HJ@YuZVUKS6gQ!l?deH)NUICU;$Faw+P|BA2RQPsYuf}c_Vm<)-^C+1$|i)AC{%V zb>XCi)5A`1<5gFH=NG)u0G3%rKVgUR$2WwRT0MVXysR)yYK)=xY-2@St?c!4V{LN! z?~k=h4&@&B8}T~&9B@Ri%3M6$SpfI4aT^8m8O4wcS{fX*1zQy$ zH5aiMMF(C$K9TX$edZX+KO%7k%tRitPqdq<=nKrMAFQu8Y*EKFMtEdzRGamON&+C&9l1vhX*aVL)=gP+k3XVpOOEb_Ii73`R_|SNy6xjsnMlj zFcTjC9dnZ1KKIkCD6aa2uu53bC6p;h15ni^+W?WWrn(iW^*#}YJ0^1ivM(F*z!3ph zNb2&}o(SEfQ&1d;fV=}g9$$T!Gj9wprz~i1k+#Ks_Y(tts=27S1~ z%ysc(nabM`rk4vR+X`y2ikhOX>lZbL22(x@3J2W~rsxdwo9Pa{|mSgR#0A^-_w&1^p#xn7>65|Bbn0a%bu{=`cE{RLYH-KHGK} zK=dmmRi^BOU~cEs;$}sWCP2aVQL~xy3N3=@NuwrhYRtI9(m4wsrs)Fj)uP6@Gw~?e z2!tNuA$2)KBJrB-=OTR3u_DV?3=OLci~4?Zx}mmdOgNIX#AFsL-ovSdD>7gxff4HI zHxQPF;|wA2GzdIC4ugk=Kg(O+`v|Eqmj3P!3sQ*eB;PT~p*u+fL5iUu%Ng1|1plX~ zWhon2!~g9Z9P~2$-?P2Fwf)Bzc~;-NFLBksF*ZF)aE|$={;OAOD#V{A7-R1 zTE~#Q=8II0b=rVfHMu;n_oK<>%lc(VbCU(EhUBX)RTWrYd!Ls*tJ8lNo=x5QU_dqW zfB)Hj?*4bbw~qhwMV>;%D7xyLP`8K*Xa}Q?!;KC%{YY6kasp=E@ckaU`Q?}EXFI*v zLFqXqI|utlJ6&q`;PuVrtMlFGiGt&zf;+v}H@`gF>B-V@*=M`ivLB~oRruf_(fiXq z`Qsp4@cHh+v!_w1p7BEdpqZ|r+g}b2p564XWYY;yUsQtnvOjope%wDDCTi@|)VLhp zT*bvFlcM4{wKu1Gy&*YEij^z)d zyy(qQvSCyHU=^j0lcgusrJuz`cXz6bK2H?gt1kK?QS_j&=*iXo?(^NWL2d<<@04o(i9?eFa0 z?0(!mkSEX1KHDcdPt9lYhf!GJ*}*~Y`OWUfon9}l@$8^ScAobDyMD67lp8m{F_Yc= zbi9Ai>-BC9ux#&{Z1_NS@f6VZf|)lz(%|NrdjS_WiW|H^m6EJMC;Mcl*XzNLf*LzF zz0z9yJMcr+I-%a2S=P{6;lJRb@Cx7gfmwuioy(3HS z6qbG!N?zw?aQf=SL88V}S>tI@jh)`j;B=?=OQP@#S@=aw^EGT=q)-Yfs02+_Uojk5oDJv4E@hzdUhr{(pOG|GzKuB;|kVx_+z^ zL=s>#r^G_)YpHm(HKNBLqPCAQKD7%Bnt4LpxZ|k5G8J7iL`fidw5bXPh{jkK3@^c2 zglN21%_ruAKqeqZ1lhFssaU)^{X)k#j6>ovNkWZ)nVtErs$iv{t%a8mJrDAc6AB@a za#mEy=*L=HnTd&wGtG&u3H_GHZDsuF=@*?T&2H6=<2w5E}r#DgUtIH zt%m*t0(ElX@nA8qsPC(|o){9T1Hoktj|beKRyzSq!aE}yW+QGm7l0$tK>BzuZ=;j2 zeymya)Qc!Cwx0|fygHrG4?vo-4aA5<83SAWs6Yg{1SjP?NUyIE!kP#tU>}2YeUDM| z%&`_1LIkh4#c;6_Nlx@bElb>;dwk9?BYtNaROBsm0&X)E1E!T6bS7(ye*Ig>{2&Li;f9A& z0JJmjZDtE^0iRBI;u~}3*+oKR`MG~J`0?hne=@v0?+=EV<{=yde5e|ofh4{`9S zvOdnPFPCo?V_TIqd2w{oU%pKh`Ko#@j)vEx!SLq%`0eY_=?asb43t&V(?$R0=wfs| zyjZ0h9es&>iv2lPiKW#p12O^L;AnJnadvi93t$N9G7R&S?24|glIc*Jksz8v0MyzCEd&M(e>`PaRsITg4r3qhHyZzjHa(#yAK0B8+@a(Dsfet$4HdwW`c z(k%mOT3EZ+B3JUED+B0cO^0eQJz~Su!PxX*j$lgX2`OtHh|bOV+v8(J6w5K_SHT_9 z-fB0HDARp>b;kbu?5LF1KV1LCV?R3@exAO4ml>etIW6n^Vt734Uk-1shZmQlv(s|a z(dKRG069fX#v;i{a^D=}~1;a20jAYDkqLkfYN`bNHI$KR-Jj4gR$( zyIcrDDj7~(wvv^Oq_ZyuSLEee30ME4*>`Ln?dn5Ddq@p zsotNWHbV6b6db1CMWBI|@Bdwob?ORg80iX@6vvP2%+?Ga+ufWE2nv8AF**DDiD- zS!60YhQ*l^kTG|tw`e^gcQbCxAi)5Jc~osYR%+qLm=`jg(Ps2x4- zBejT_BofI*nQhRer}c=i?zBs^pY?X^PaS39-GXam?nEs=_8|i#cMNXHf|d|21T|_s zk|Uop2kOUZQc`_NxG+(VFUM=isguQLa^gYGJST2_R@^8rZgo1}g7dR7A#Risx8RHHvfci=qe>CL*yEb8*@S6Pb~s6a_f&XP7Dm9`vT!0cifE0BAc zXe?#yqFqt*(@VXnpKYVlDbu1>y1;_jNI<#$M^Z#GsSu#*SE@}%{$a3Laaa(SG+dU*sXj%6% z6mG_{e$H4h(OIYtRjn308d+)ErQEPAC4t&{KkfCg+5TK@77sioo0 zR0m@y0Z-disbEG$-Aq{!PnovW#(yyaPhzY}b}57@PH*2egvW&6HH2LTms)I9C$YlO zZ7<-|=Y%)6s;SKKl}4}Cc(6U;4z>92%qxUH0=vsuw~YJtQo z--y_xfpU`}96#`Abbe*2fL}@i$`>nhqE&3IY9jNX+iB@hB#!A!a~X0DNc4c)HJ^-t zelOd!e&xS56Z{|i+)Mte3;d^^y7+%j_pZL=I@#KQseUb2l@-TYP<^G0P$fXvx8t#W!!utm({8IaWW%{3%-71fOy7(VY_j2+7 z_Yc2ln*G1K`!pN>K22B6+$XE|c&J1LW%)Xe4nsfA<0Xb^hrHAU}`79q(uORV&N1xv2-(g-?q=GWBo z^SG%#)BsCT(E0%FMxkfQX0sv0ufI0c{OdeRp8wq$vuqC5X3hJkk+w{abJf3_lZAamPMwP7^7Vs zccxp!V-^kgoDr7>v&4Jqn5{>|VN;;qWWxoARAwSg`u}Zq$(dsY;U|LM>W+W+w&ACKF`I77ANpT^?rg^m zjj3@Px@5vF=C_`7{X4hyq&ud!ttZ{UmVXP6Pg_s^|E)*lntEJ>J{cVi{Z`lYgycV3 zU2ZbEjm16jYpXl=4PmnF?}1D$x&JSQ{iBm%*EW~wqk8}EKHbade|LL(>->LTMYofx?63q{w)&D?ZL#tI^hPa-XbCdamI)qVwLGI`RND3OiZ$<1L1G{?6Y1|5C zj97fajD=w_A|@Idg?Q1mSGU!AR0?V-cov&8Pm(V3?oDLHN(L84{`-wt5V-!~_IBF5 z)f5J#wlJ6*K#W`Fn4P%DHg$Oi-bztm8xxSmsg}1lTXJmgA^Wx#Tbrp*!`c8G(nPo{5W*nb1o_GTOOjDq9(xjv6!p zkf7sUTB1l`QyL!OkeToUS?{%SB9Rl?p#fwYB!df|OyuHa!Dw~Jr9S#_xnwu61as~M zA9W^ z*Ax>%eP0+H9AVC1`A}XqZcvAeS@LTjP$ECoxHDd;nUsgT1c17(CtOb!C+7z5o2JKn zpHLsZUI-Cj#4R~w`6k(%6j#ym7k27TAvbO>10Wuc9z7!G9s}Tg9x%Vv>Le#=zSG@# z_WxQPq$cw{tT95m0WVuE<$@HG7)4fyZ+P4daKE?<^teIaBl&J2LXW7JeXS24KD2%y zM(A0jGx0BvNk@<=3vQGv3(@=3qjz20tD(<4Fo|#i*tW7H{y;PMZS39Op0c1L-*@(8 z4>MbsI3QgAEoT zH(15j5k=K;%#X;x0^U?HeLs*X4~PNp3Ml*%ut;Ph|4j_D%xtd}?}liJ?zaSYln3{l z`R;a1+p6bSvdx^`DO$+P*hHlD1Cd9o5mdoHWdRy`!CH-w@)*=-=p*>y%cV1_KZqSBVe8nlPEk>5f92O;lbGA1(k5h zH@R_YlB(Sw9mY(*^)n+gI+w>n;0ZG{ZM8lT2$J@RK)jYuM7g*?yg;O? z5YVIbiF7)h4*Ar9NBv6u1^=}^kq=qVybt6Pm}!zvCQfHy(rHr=%2mt60`TPvcz`2|8fXRQDNg%7kC#2X>AIXJWb`q1n zQh(up67Hgi$|=bIXy39$`>z3MwZS)OmZ?(}YtbN#y-NM>`Zq+rJa&23t zWtio&*u^&)B|1IYWI$|LN%{4DA|L+vGt-DH8igCrG@}Fe$SO8<-(jYN_rBJ4s@4V| zHxMXIm^0^|aA4ofsmCR?!6P3I_Q>N{`^{y#)ZU{RsM|>&()QsW?InFj+lTM|Xrpsw z`*3Hg9eY&D#|nVJQRgBX+OaRNEF8uNON}W8-@f;_pB;)n5xy0wPX=I zzHlAkNb=^4sM2K4DY-hn>_XTEY?|1iAK(cLEQ+iL0pdV(`>~l90uI+`hiT~= znT!G+bLl}xRWleEc|GZ|CivKBD-?`Tj!sD)%*z}h9=`~lT1Ro0B ztv?$UDM|Ks5_9zxUNMeJf_%%7j z(JmxLqLfnCXoG_Bg7|kFZ%&ACq#(hasm0>f>h`tXj9v0_(NZ=RD8hukX58fnU5_aN zF{zT#u`NKvfid9r)iff+|UBe6{yk=~Idy`T zQpy~*o{$e1i0K2_fRj8P*_e?r_1V*X!W=_Lc!N7`uuWM9Liu^Y2mSXgTYPA>`o26d zK?MNnu!-=PJXyGKm`QA5GMqt>YvWt6Jz@)w!FS}_5~G%j?1wfo;)-ZQGu299z?KS> zqYz;tLdVpU02QBOW{JBl8A)t#!T|2x5FYFq){1&bH7@m-oT351z!Gb1l0jl+lyVu+ zPQu>7|Gn0eSV8JDYeIZJb(+_`beX{?IA*nAT7XL8MHFEOIkC?x)l}}4Jl%L@(J4+W zZb#IF2c_G%L1(p5*K%B0+V3aBB_MEt@D#00;+~cvlDupI*!O~3M@maWtyLEhMepcx ztH7_Bv5SZ=(1u@SBU*Y$1=0T+qeF6wi%A-{OLoV-;M0+>E}1pCgdN!!R8>_$n%vV( zpiUEQ?kJL#-_Vi>$t5*W9Tu^sx%1#TdBl7FY!$C||fi5Ni z)G=3SAl{8C7}~1y6w70DkQJFUvpDFdb}JsaN$&We=}s25!17ChbHlO4jc75lq~)ss z`-Uw_fc5xXLiN?5je_jnXW5r0*Y}0?K@s7rfd7D~u?qa4VpalYB;*ZJ+OG4I9K(Hi z4g|U(${k-FE>=R~QAE*oEx3&x3&qDrBA6s|=vzSyVY|YS2TOc!pH;B0aBt9nIV`^c zOB@p);GFmX{ovhcz81lSjV1Zg=F1GdYxKq`)icT(749nK-D@*$*(P^DVY~}Uelb-S zRkCSVuGCVk)LGK-yLdve0FKV|FpwtNAZqoHe1SNzMSKR={ct(S9lQz@lUPKCW}^6< z9^Tu4^#5z<%Y6f#d5rp1Q0Gj|)7ujV4eu)EB&{!&Vp^9l4zNTm2#ngWBdJw&2li3rs^47)ys;QstS@R@&cT-5wIsuNUQa;#JQ&E!2MWz31M=hW!y?HypN`( z$kRnjt<{R^o5zS2&6R|S_EB$LiASJtfIuZ&I8x{rezK+`0*sx=OcJ_X@=AEHFZ@W2 zhZ;#)jv5AYL8Gt*>};fc@hBi>VoVZ>2u}ffm1z!yU@X3gtf0avqMHfhn7q+Q9!-fG z)v^p}a94q=30t2|B?^>PbT|l~v1wPnV4P-A79BYg?(mm#@hWR!7DX7x^<_yJl@(DI zx;Q%NCrYKB9Ms7~hY9f0W^D!CWs~7S$2qp3pr%qbq*g__M-`OLsApxZ()*Lz+Nl&~ zvMKq3T)gZL2xc|I7J%cCyIWt3=xY?s&A`2TBnBmM^~sYX?nC%B<~@0G*lL}FqnX?t z%BVyh8Sy1VWVnJsoncoblH`9tH62%4EZnP9DbIZYXN~U)$LA1yG%_NPn-8LB%e`q! z;3-DP_Sl@WJ0KUKV=~VVW65*2L5X9b2eD5%?%>rks$I1OSzZ{?V?&`s2l z$$dY>8yJ(~em7{6%}iR0cnNdJ2&@>IrN~3)Q|nV|dOwlA=TVq8mReH?CJww94qqQyu`Xs6?)Xl0}mhN z`4L*wEBx#m!exgfa(6acL|-?muqHSy^BrorGl~JvePhOKs-tNMmpT3O(e>VC>NQD{ z-9z%JlZfk_9SAYyqGC!11|kLZ#ys$-w;(aNcuZ;=bJw-FiYba3e zjanLJsx^p?SdY0L^O+Mw=OJ|z)ZI+em&}_dIB-KabLIt*iDK%2F9EjgYcp@mBMsR! zJcn9%WrnVG>6UFOIT)}83!OxJc*mB7<3xB2jL#2~rp$Wx8^X3jhX)I^rIPC)Jb&9{ zb7pP(eA=O&G2;P)z?tA70|SA3Q@?AQk79%0WTzP69HqI$2iSf~ERWoel7*sLI>&3^ zXcaWeb}_uXiZwH^Qc>~_RWvs*dVS<W(}y%zVKRsyH7_hLZj76(*0~S{kS0lFJ#+zzXhA3sBaQ2m+XbnqMQ{(5}ZZ=jGQ3RzZ2SH=9Tq| zt}chN?Wg;dkV$ekEzBfB6h<}z4r2}^;KRCIaw3;Z?MO>FQ`qMvGa;X+`yfJ518v!> zK+kQM_?Z_SMJ1r;s9yZ2vne?&W>Y8NMr5o+4Ff`^PSKpuo-?m2yeaHQvfJ%-du4+l zWpnc&b5KGinvOrg6u`J`>dMphw~$(V!kHQG^~jd8bQ`>3i|800qap1g(GnUUwk44@ z9Nsk?V;v`R_|S-?02wjCEg92rLj+GIC%+iJT}{IF5`wm@l*V zY9e95m=aPU0*7isQ;I9xk8kH9v5)#mbJBz=Fz)6^71a?!&@LKUPx&-(3$Bu>$D$5d zWE!c#ytKH%*79HUtmVJ8{P(@de+6_0L_m%k4!BhyJEe44Kji`y-fkAiSv+8Ny|7op zhb!|BmvTv{^u3%4t{@z;p<#W6#B399ENH_26P0a+t>djnk@&lj6PcANY_nTPUJIKq zA+R2sQqQ!Q@1t8PI#AHOl&(D?!6axU`PM9H^1h-`DX6$Iez|gSyg=Pzxm=W}1mu8P zRy@_YcIDjK%$rJ~#4!srrs=0df11IfE;&=!QwC<;#FIt|F-s;sTM>z{HbhCQW-Ho^ zeKtih+B89@>zEoQ4Ps0-ItaCpLQ5lU{^Vhfks zNqErDXkOOM&WEKeikWN!nkYWE(@&Gt#^;tT>WXCMOTwY(FwTp7fk~QYtl2W-gQ#D0 zD(0VB6-xX>c}SNVRxKJN+fFbFxVpfmVuDY9#$BMx`qre$g5h&bEpA4gAij^BEpjRX z`3pVV0SS%-^G`(pZyt2^s*=(spXKKcTZRqe5UTI-lbo90C8J5&DwO*KLR6?JXwVHX z#W!N2DI@TzG+MLp&CJR^w5Q2Jhf#%CgP=a}h&6k%hW;eV5tN30PAa*FWS*2fN`<egePm*ynPFsX)^q;MzAoFaapXrY8Q-DhcY&G;9^ z7*`{IBP!UK0yZN6rpM1(;-Tla{-Z8wO**;m@m=U5tI?^ zCTxIg^GdkF64OO;$}yl3w_QuK6vPk53VXoQFSABpMGaK8)Tvn-;{-phX814>(ic|q zltO(EEl(Z2l}&w%TAr(F^^R8FG?6a9AN9_~*VF23rByoQb%3aJ{Jp4m`u|Z%or2C@ zgZIm^3az9f*wpuwXdX-jBj;5m+v;dQ5I$qf@WR*iSb&~}5dRu%;+mjhV_no*)0BDs zdYYzJ5L@eO9PSv^Ykjz{5BKb|KHPK9`fy(#?(4&SeYiIq_Gh^xsDpP&NXr+clUFvC z=>1AY*nDi?j@|vY@~mK{zFnI8_eRXMZFSN;pvM3E*+DOp|9NkB_q+H1{!X56*#19> zOr*+#KOzaHT{%p7sehpr)-sL zQ_MDkveVi1rD|!5@CkU0O3_-r23d!Ofv{=74Qg3B_n@j!beBne42E|7ikuHdgyL0v z32Rs=ff&@K5Z0(rp;AA%L888jw3!HxhGdreI!#uQRF!HW3e0{;Hf(Bq;;z5BMZmT6 z@gHrQ8f|1C|9rg6Ac5RXsKKsend(u(#p}X_6^rW0ITrH*#Ya`AF12g=8lCq~G7%6H z!Ak2VucCLkuTn|-OMy6vOX?V%Wf1W=D_X%q8Z*ir=vakOTngSGtb;QM<8bZQmqg-8 zNe{)|@M;<(exs(ps9bcHjP?7FS=44u5Hok#u%^q5Jn11_1CkuJ=4UbqWc5LkN}aR} z7=af$w{?3%mdT(;?UFQe8-k`|{6j5sz9%Gw2y8)VzBAQ(b3n2fjeIrh869X<`yl81(8CdYl{NKqQ1|k4Un^XxD^8}A7Zf< zr<2;}jhLN)2BF`K-GQPhhDVH115?ZK6!yNXItSi_o!(mi^Cg~dSpSnnrq81T0$<|M%VIrH`eSiqhZw?)J}CY5 z%jksC0GH^6GGMQDL&;~Y8(QmzzBk=ade_(MhYDz(ekwvIe7njN4l&xt)G=Sb9UUcf zD(MfasaL+Lro-Y6Fu--f^Ytqzs+M>Ivn_m5*%m(0Fc6!$2AC5TjI0A{x`Nl(X2A#5 z%jZ$x6r$2@3%J#F&sK0+5&E-WhEziShX}*|fnW2_V z#LEhT`JIcHHL{%g;s|qvxPSVoiT`)7-#f_2fBVn&zIXX=9sln^W{EUL2+To|d ztcdrQD^x5P7JyzPA!ds$7x?dfl45QoRT5*_c<@;Zk%?z5M6QL%?^1}&5rl@qWHH^; z&*v~ai0;wUty%s$EegkW{>m@15TBzJzhhxEQ+5e?e@}$T{_zjY0c^5)u3q&m92`8{ znc^fg@R^Q=5gSkVQ7h>|6*g8Z#1Y_}UFMy?D*tvm8kBsbmR0zr#@JiTn|O&=T!3bb zNDAAkcabT>scL8x8e#fNp9PX~m0SBaj8#Gd>e66VY?)C76RJg`25!J;Lq)W~C^|38 z>}HY=*WyK1G=;;D=&B~AK;W?nvT^<@ua4ADGtwF<>z3P71u&hzN@)HX(TVtosw~*t znX@rbB`cyZS=Li4HE*T#l#HT@YeI)ef)>>lZAe-Pn%7+*yT7Cxq#cAeN>KR1KihsK8i`8qR~ApGCr^aZudYHmn*?ldzL!ppNM#m_QBX&kNWzUN*8 zbH2ilcsWvLlRcrK6&w*Gq#d=~E)t{K z^9dIn*Wm3gIYMB7J&9`Nxxlj9!tx<>i0-BNB@oP;KV@HW`q!cJoDrx{zL0*cV^%{U?j0PkOI4jLImIMDW#;}ib4Wy=6 zf`2GErwxV5o1d;dLH|Tt=GoL?PCzzvJ6wgYjf^OWBknfuw|c7W z$}){$?!*w^cha50EA9A6%{gw>!b|HVy9FA&WWP}7ejPVF%Zgq4D=H^6qje?=qk&3( zL1gLNsbsM!H5ToycBEfzH$Q2xa>0jkWo^Zzv5$ronoI{?tQpu4ORE#*u}@Clo~ZCb zcQfXsQemlUsM2W1ru z!E=eQI|t8p@%QsF{O;}Ssow|c_cQf-T!6l${-|f1*D}-0BRXP(DRMkE$_L^wrJ3nS;x7cU-a zl;4P!b6&?mH-A)g-ISh1_JoO62Kp88e0zi$2J9nh&MS<~hNLWWOY~ zQuY&Uh0M#(V?CWW6KY^Zotc&fc0=E}ZT0u?G`IhTWn97fyUzamVE@_vZr1*Le{Xkf z|NSMN@5ugJA>fy>{syNj_+D=O4P|O=zY%OCL#3AAzUr>t@H@$!f!)GDSfQ<_xz1W*&*p7h3onY$3Xh5C?8Jm<-##Y>V58*O0}k!xSLqt)METQp5%Yag;kcK9Y2~% z-Sv^OA&o#_p9caX^gQMS1rQAI%gPXdzKjd!iraGSY&X?bi|RGE zML4^@M0=>=$Dq#s-Vl!Av7r1avex>si8(@@jeceTSiFPA`NeS88e}##$82u2|Jls; zmonU6!Xj!3JJZ$d;+C+ATfv0CzWa<_{Bjf6z&fs(A^&Qo{7d4*0~qr+G3Re+(7%L9 z{}NcTgeB*f#n%Tk?!UKr|B?wPYKkpoF_R4bTT6c%%{4j*G*3`flgI{}Q|Cshqaa)c zV9|~9SDV$FM#+?x%J8C2Q#6tgPL$-OqK#$qT%BECz6Idw&hQ#E$)50VlE=fYtliv~ zp`l|&Zj_m2(E>wkNPKpES=*u}l6t6>7KO`>j4bP^LGEE+PL{PXCo6buG=*asVw42E zk-a%t(u0F6@5S-C$JxxAgDm6GLB1;PJdk$>S;j?yERQxzd3unqJEQepAK%PwY~}$% zmi7Yqd>FG-{;@{$P{#-1Q_2k`oeCe43v~_NM~j#xfpZO>w54fyI5vi?Y09-2{>p+& zakNz_)#~8tRM~rfE_><-GYS^5lsT;9Q)>KXbfFRt6~EX;#s8u{Dpig*_$EbbxM3jP3d6LFT;Nevb)?*7bI``N@jV?a6P@|)de&%4;txiAsxh2;h{Vd4=NIzZY zzug(gmYicO-5i7TGsh$?u+?{R6`C86R=1&ANZOo-<^*MF+iz<_jjlx(2sdSsj)@?c}HW6ZL&vk~dT{~}=EL%aCj(!k44r?1|*}Xf)RBG_6o(oz7 zQ;mmFxv3_)(NDwVXB%qQ7HGTRYBSq%d#z(^w%c8x!Fv1KMH(BOV%B^Y*s-k+Y^%jR zq2aX9a(3wuCZMWUIg=5o*ioC;0-wqzt;alCj#k9lSOqopCsz(1jeL&10qR8}Ce=yHE6bFoVk^s(m1Rm>_bbbkm1RohC|8y#<*}?RQ)-@-WlA@|R+cIG zXJwhPvP@Z7rmQSeG7vo$%an#|sioWqy{*igpMARSg*vf*8?AICz2-1p^EcLR-D8U` zVB^zz#MJgBw_yU)ZuO&|4u=y9>|7Q=otSyXN@%W=4VqhypTj0dKl54!b!-;R8QwSh zbh8rCPbXU;TeO44kS*H5ZpaqxU_E4ucCaC`MLSp$>1S^7t)o?ue&(_*>P#D18>B`H zqs}ePYl)e3BIl@VR-Q~2iER&aZvhD~`*K@bvkZ208s7c8(aGzg#D-?cxtQ2*#WI>5VRcEmPxH9`MKP$8UVxg7U zf8(<<`(K&;7mD4;?7vwgZnyZqe_jK-UO*X`D=rL?YUV5?ZIHCIa5MLX`eVU;c7140 zDDW_xCn6qUe|i%jnoj~P0NlOlyN_jbHe>FA5LHk&z?%j%owM4?)_!Gcf4^<*OD}Ib zxOunu-y!F|p7BKjfH%kg-hRHl^Sl`U`^7igtN7n{@jQ_D-&q2@{Q%%6ocnvJiEMQQ z@U&242=EN#dbL)@08dMG3Ie{=IMV4rIs}p~4o5$$NYa^S6-jy(N%|p2l1}--e4(TZ zT+=@F;&p(pe`a`!sgH+#g&>I8?08gu?X;YQoMwTn-AX?XSAqaSmE@~9is@M$8zHZh zKp_#XVmp<727F`^T>y$eb-x+oL&ZZ;G{F&aF+*dHa=pc{k}xv_+Fz?^ao4OgAB!=p z49w8B7VZOr846-QqM>gSY>l~UGA{K}b(ovjm9c@YCe)q40>q<{TCFo43#6u6ok0|S z!DdenN{|3kruuw{{Sk8!2WT8);mN9`=4473qNC#UQHjQo-PGWTvx=;c*N&1(1S1iV4rP(Ug`?>w zBWR~ja{{%z?PV_aSx8(uqC~Mw9}kIdvY~N(A#tQIH|O~@z>zySxK?ab)*Eo=&C&~1 zGz+(mum!k`Vj!sE(e^oq`A^FsEC|{?+Anig{dD&aZSbPmZGhBlO7@FhYt}PG@537i zqf0V_4IHEdBZ))d+D-)lF#st3lFXoT^}cSOE|)@=q{el)0wbzdBc|!`s~0dp5Y_CE zaTxlvf|#;Td3Vfx?V>ieSX-7`R5A;xV4&yeE2(j>=ygIvxC29y0L3;>fPUU$pL$92 znnDlR20G%A`tRVHDtSRo=RPO$h)3||#`tMYE;=uEbZbD|OzYIt@Uya)?4v_%L!Drf zN+r_(cTq!arU)fm>rQm@;7K-A`v`ML5Jt1g_6_#9$RTgRu1q)QO4`$3v|#V6lTflf zs!`*JjH%n$OF+b!pinWqrn&5J!QAl{rJL`X9@LJg-D3Qb)OG44^VO)E zT+pI?p&`L8^jfj78q6sd}+=Fku{@fdThpT{^k2&%QaM(mWd@6s$!n?wP9BX}eorXTI3a_bVN)64ne*E?f z{rJoA>sG^$QIgD-cngVc^_O1Vs1*`AQ*}gUo|AI(Oa+eV~DWiC~S?Hu1P0ancaauk1 z=~>{KYO~v&VR}>8a{sse&Gt8i`@fwR-@JI>_kR!Mc_8 z9q#bt>b@@ftnTYp_jM2XzHYu-x{h~qE)MaK`n39_(|$hP%hWeVw$S;BSIMUS6pex%HiTP*7pELB}PQzivgX@iHg^Xn^fUK>i~c|Eruy*IgctGO|A}C zj=jO=BNSpr-3>&9;9>(^VZkUHZvZqK$Rope+(-Kw#u`kiqKJ-=Peu_M`j}M^e;pM< z?^R@|z^Qm|;H&sji9SbT--6_;0aK$+@jfVOIeQEd{THl7opfrVK4o3y%7nAY+` z$`BWx+WR?jCzvwmThm`6n1LvohMGZwM6qC%wztDs?yhOB#TQW4ZT2)$mp*%a1;Ovp z_3!Z7h^<3{{^xg!uUA<^JT3U1@wvF5$^(ds0wxf{@bmc{`Mc;H$q*$?!!OoK2+Rtm@m6!fBBY)`v*Vk zf`NbVvzyP~H_Ms4>WgqE$7`0ZobRHNuRmzrTh)7uF4yk7i#vaG(Eq;JD*1nGJ>PoZ z`rnoR$5QmaMFPAU!qzmTmZOAH=HkWhbrPc{7stTs!`}?r8>?$E7kEz zb^IVz$9YcBQG;B;AnntjNQT1um;{_1u<2VYDwGgi?Q3r6*s)iJ1!3RkSFrN6asJGn zb_ttO!5Nt2O|hVO=vUfP=lEu_vFl+w9`scs+g2LJWbt*>=w`5dq#fma72t=l$^ti% zq8fkxVHuTC2g9Gr`FrJKP~~cMbsN@I$ygG_vcP)COYEB>o=JUkcIq4+zcu;>aAz5E zUS$JoPC0Pc9-x|;tvl(}6n89a^BmiZQXbrCTF_`QW~!Kp^uQ6vm{NnTbX@PcULEQFiV#tiWfPzJI=3H^xqnJu#>)LSuhCx-?r}ohq0c-EC)S3P( zRB<1NwBaU<(HLx!sgvWN{lrw0c}Z^}VN)F$siMi$tW>eM8XvtTRE!8#(##LI(qMSL z&8RM#QXZL^4l_t4*C7#9ZOhoVK$Z99hOMaSR<3~G@4Z#J-lL~ZO&MUt0T^a3NxnJoZF5ikJC7!~S^vAWwX?IG z*Z=N3+u3>O`rilgWOcvOoi!YW$)8#awpK88NXP`~*0i3oo_N-btQ&?t zb+uzvqW|6MZ}qny^kRB|&jRb;C^x$^fR^?D@|$m-71#fZt=0O!lPBxIPXlZ-U#}y_ zL4t%_QgAFZsuZj1O1(-08}cMAJV4`#3s*b212HrS<+YPvdz1Cdq}X7jCC--iB0L@- z78`jwQ$37*ABB*@y{SI@I*EPXWP7)5K}!+$Ng?rf2HNzi5g@=>dw_cCAm#|}lYZ*| z3aG;}BBHmJm%9e2r+w}12Lgdv2B>$$&)f;|V&Aw6wzY30EhqFbBa3ze16Mud!w*?$H;f19#nx6}{tie0EMX5PmOq^WZ89H9uvebKzx*gjJ>&NWgVXXDW=!%h+r!W`lp z)vP5o^;-ww76q6TLsws&Kz{}RvJfT3OM_3^7!yXc2`2GJj!y8Gv_WB5;7p$FCuzY} zJ|q6s7cA>9|3P78AGuKEt{KZnCRAagf`w06o3w%|0ops$W-V@OPeJs32$WhPt z2R|Ghp}m9C^TSt%d%Ncc@U~R;&Eesj!^6GT`+Ix4-cRG*tHbYi$A{nV?(hEe{doMx z#oA+PrS@z4FY|M_^vsesvu0U;XqC{NLly{q~1Z^y|US z-sa}>;HTZA@AqCFMqAtCpYi!&$iM3{h-3^^{jcR&XWr)91{;DvBoa-CP)tJzB16tR zC5xFalx5G11p}15(>0!ZGq4<6Git<(<@Hhp%_;Hum4RaonF{F|f>u(Wh+-^{LIVgcVifBq~H@p!~t?hnxU-idh^VKFAr2^6)J zwPIn(`JY8EysdQ&x*oS@w`AS;TvcUO2B0&hcq0QeX#1Dy( z{U~fw1+<+1pS|3ESvdc{c)9a(b^gDL=fj81uh$N)LmL$;n<^ZpyTA7J=BH0I3zvXrr$x_K=en1zU;AvOUy)tM?C8>3B+npdLXVm1MT zZ=mgla8)6XbUI29Yd*yJT^yIb zlYvp0-`1G+qm7-2djGwLdcXJPUuQfBIMd+Dj^OIT8r})Ton?SNd_a`Be(bdj zwXcC`DQp!_uT6*j40Rby>?U^_>KhKjg8C!5fcWsCqRL0~N6aHqz)ZOneRA}i;Y>GV zDb2_hm6%Oz?dHO5amY?3JX*AB#h>ON{pvAMCL>imKcej7tXc!bZKR+JF^byGgUza7 zj_UYbzF6PYkQ6qfC(Wox8EXG&UBfIp_6(m%*r55*>(oGrm)!mSequ=@mSU3yQK6pf8|7BI0z22t5lcqp*H3~^+^m_hs!lVt z^nN?5;mf%-9ZDG!k#l0|$UZYX1AY4BNaByQhS8zAP2+3?9@!->svWRDeOg0i2T(l& zUK>00lfCXEVw8D=Md-O|##;W%jJEQ_r?sMwS1lg)Qrs>D8DdJ=7*xHV2M->TiKg2@ zU(GuJtE)CFyk7}6r~y8G>dUG7)!lRh1ZIqIX2>5h~;^eYKZfvKtUl#H@oaEF#; z?TpVoloI`awU)hd&@(^xD$m?mB(-vzuwg?_7>~L)pJzW$J5~gxeufL?>}Tvc(rU8m zCA7TZcUw|g8Fd`uKVq_Gp>c8FHL~~uOjqL|mDll}h|otgeEFO(H~S-zoIwq~#4hz! zK|@z?;km4?ej>BVuI`Hrca!Hv@?UzL+Gq{bX8-f-Wzqic<+B$n`=2{`^1k1pOof5{ zWO8pV3oNkxLz4WGp+XRHMp!gJH8zH+r+M}!x1uFDAnMb4t$|UwT8KsD@KHmZoGnG= zxw#t1B}hA|?LrmfvtKoyWzMUxA{E1;cJya5kvvRw0KC|$S*)^Bt0hySf|J{1uE+f916+>jb?j3mQqV(+W*ioQljGu z8c`Qpmy@jB8VoMt|HOe`Ca+Av>H!t3z#5v$Wz4EZjx_^oY$q`-t8Alur*S(vSw@kBUUXPMnnXZ!F>E6 zuuD#epxjIDf48?X%N%GXp)w~iyUec=mj?mP#td-KW@#mfvCI~59G_s12nS{}WA8G} zmeTzlpB$X-o*$q7ezg1M;Ot~~?;vA{2G{(r1TQ#V!VOJwh$*~3!O>)pu#P^QGs?q3eTfK< zGqhT%0M9V0QC%|yOV`q1vPQ7=z|^*qk(O<_!4g@h60EE(*h7fM!9k@L{1vWuvglvqd_b!-=3% z>XR`!kS^9q)@Brho0s28 zCy(#VtUgVaCdX*mQe0#2nEA8ZmQ$J&JrXmBn+(CJfsl^2b^oBeEa51hnx|He|?Uq4qo-*W1JFFW-Sa<9;ZM|@)1xs~2F$&|Xrfub5)b3st*7l)xN_pD=qNm87o zQU^7oF~Vsf1Z<%b%0>dqNW`uUC$vCMfzDZ)57gA$Z>~znxxY-RxhXsgqoGPXtm-Q% zR}MF;*HETN{SsXk+20)Q%@JkTr+*SrJ%s%C%xLOX#1cT#acZi;NxoIga(TJVzf;`K z7O~_bf*QwzIUvllvuO$laM*1f7gkkfy`-dImRbdm<-#-f4TcKJ@W3?PJ}e7w@}FB2 z`z+6m<^SJ@lzEiZYkykh|DEUGY?bVPSMi_j=2=|!|6b4glKH!#rpL>`xlhcErR7@< zh+ng&nb|Yf!q%BtJb$$&V~BwVyNNYh%-%>Z-cR{f`m2uH_JT6TZIj{`EpaSHu#cTm zKR4fA)^nr%-1f408aDenK@Y=5)z|Gtx_#(q7wyl&1`EA5oNWUg9a;MZgl zQE(#6hMp=kJn#J9X#E=lg|;hzHvhk^XWx|LKRjEl|GRh=Gx})a8cSdo1&^rZ^NMUJ z>4(5B?7gsd=E+<`<^$OD)%Pj`@7uFVZ!%{#oX*&q{Xg0G69u@T2w2qu|4WxR zrjIz;PPLNVae3faa~7wdHeW+%gsC43Vl7}_Vd^JTG%XbGUk;5nRI6_FMOQ{+*{J8w zX46YbCx$jC2@#n6ymxxAd#>Cd8M$8u3ajTf-avx*IHFSmd^YnQGjDAjF*4Q>B%g$WjOaD-bV$el z_Oy?VnLk6EK}`h`3W-2IWu(8>-#`2PEaHN!ts@hU=Q?{8lG{o*@9HJ5RiCp`-FR#M2ulqxMx%PEG3e;~d=y>hx|Gl=3 z-eN&{EYacqfn4i{fMaEcK6>L^aHPJA1&+u zo2`=l*UpQVtNWijdDfefb*%ljcW-mqIo?09$xvx9?#l^8$WwCc5PM^?X$m_~)}d^~ zMSyiyu)oV_=9BM6K8_suM~nq|ivHWX^I|rTmJU>TCsVvjAG@WLCM}($6&zi@wU@QE zw!A27>xy|mtlG7YNCy;9gFzpYw36yql4%8fT>A)Nc=f;bzuYhQYwcsnlM#KiPKp=r z-d~`PT|&I7{!l(xEBa_N_kKhdA9}#?dxLi$dLWArdVs^;27KrZzJ31u*+$Pg0`&&H z^SzVaMlX$yn$-C8exs+(sl7oj36a|Sq~L6hB?}C)njXX@f&nn z5s#=0z#_rP=Y=U3vLpoo{Gk*KUE`I{*>9yk&PSU*`Xjoi2@0=3T3Rgnc9?5vV~+6q zg(ok%>bcVDCT{DB9~;=8?c~VS&2&*2S2Z0^dn%o(ZnT?l4T@Tm>Ev51xa(&~KAp?j^f39Rk{A^q5oOTEwrV0N z+?>9n*jDOAnq7uE-H0M+H;j9O-ixiRptsS}X+wI09wZv+ecI^RJR@n*?R?3X&s9Mf zdSNT{CiLCIUr_=oO2H7+}8#mx68;dpBg>77bwzxLsQf|N| zdq9B>jQdn?&{N`_n%E{Ep5B1Pz7K`U{!f{Y8RsPZ_EV06ZzqdF#b~YTUd4W2om;_d z{|N8*fcWwskz*1?Z1Ff z=*I&4O~C7`P|C)!k3~blHH4(gLoz_g9j2!}7&14l=FTyDAt>Dm36fB0JS1%QOBktxyC* zOlLv8Qm8vUI6F_pO_(Vo|0G3Ac(L=Nuu0j72wiU^cmQn@=7n4x9U;pG7$6`S58?Qg z9LXYHY<3S=7UW|APb05b;EjxwYpJNrhS+5snDrqaunL*{gl3d4zhBa2SR6NkE|_ja%_+W;Czu>BX?LJjnm<zVjuVkj#S4YW{k3NRhu@t zUc}MMtpGhy7)MfLsysk_bEF6~dbU&g8c;@qI2fSvoN3xn*cg){Um zgdprjaf(bn>VQw!U)7r7R|K@K#Ck#%AFju$F~>*ETSos)PHj?#|Jq3vfevMn!-b=X zu#nhSKZU44cSimlXgsSOq8X}=gZhaW5|;-grz^^nQq_b%0-7DpEZJ_Lm`PHS%GJZj zl>rq}=NWM=6x=0J5)V;*BUi6-)Ah);ONghXMnK)oXs}MMg18DDM6l9J2=$oFF)br5 zAbM>2CLR{dI+G63hq)~_7gSSh*659j_NAaLYS#Ze+unXw@c(-DY-{EJeK*fN*8h}f zY1vwzg1lMK^%TCAG&~ii)LFew*=oF9n}a=%Y9pf)0V;((?#DrD+5MlAGaVzguLccQ zud2D6LA`=|Ez4dq`&hXNb1LTIa6i||seRwADd0bSB?7|$HB^XZYJ9e_GU}$LJ%y$^CIoyM4dlw-o-0Sed5*hp+DAjvLq#4#qp>@HCH6dl^QPl ztkiHTHQYl}!xh$jtMaXKU$alWk}chQ65d~_XKFkB>{hg62$4jv?2`g0*?rLAzC>3O z>P{fvwzfNj_`_9GSo&zjFL zs#C`_MT%+HQ*rDr5WyZap$=e@L>r+0lg5His`H$-`hjzp2PTaznwtdc zHA5b9?h{=6>IqEQS!p4apxw9SjLjA$5>5CO!kI(jkm%-!*H^=0yh>RV3O=PC@iq|S zy0{vetsfFG;vy)su%;ELf%Qp54uUY6?Nd=zA~QL~lm<0pu3V<-+0S*B(ZZ?8O!nT0 zvY0>!HFL@q1Quz?J!>IMOcOQzJb~PkNHIjM?nuxK0C-xRY^4{)EDZjumRK_#AgywB zE7U=Ypzf7wDxKaMB*2(FUPCFmQP)?+apo|z(AYxm9hfMruc#l9Djr=LMJ)lGlv>Ly zi;;PwB+GP|4D=^FQ5)Kruy%0+uBtRR>QQuM+V}d1Gk=z884Qf8K(f>VT@f$WQ_3zH z`ixnoR}AVS<|J`tH3)@H>RR*pm?~O6K2>CzEORs@(G?-gAk8p**-y&v9_{BVsEvj) zSVtD!b)#SiXBro?Y5;5Z5>%v#9G)X04pgIwT#=po6YeFmTn8SinUtF1%{)*) z!ThB|J0Z`nvfaXO2vT{Q-o^1j$TmP zKAtU7k_?Lls<(UK3TL^14~^JqVh4#OcghE_jvkO2af4kdP$J-xXIOWRMWp*^O@FVr zK!K9mJrZH+LsR*_^{N*}^K~X_zS2Z3*P4y%z$&{IIg=cj zx{3v#n{{v3c1pL|8?aRNFNt)hlC43|7-~)jQp=4VQA0rlC#r) z*59rY7Z1||-pgmmonV1o*ax@(n}7{8Bx4Gf{a{?6uR8TP5HkciI~E5-P`7H9jdkv7lXgq*^kFRzurej z$LA(&F$!WoqM=XBS}|^da9Ykx6?A47PF4Hh&ks%ph4{3!d_WX*A6YlNt zAoNL8(#N7sOYN76b~2j>Y}r$abK5y=4dDBlRxvw;VSC+VJAGQtoQ;#q+N1|9?{nKz z8C%V=;oV9)R;g}*uE5iy5N*gcR^tIz{O^L9qZ;%%u8}5|DF}5CgK8)7PUIub)a+*y z>fV203ACMp6z&TgRJbp+ugiCUjqO)iH#91&p?pqd&4=31;ifw4^s26fQ|=`NGuIjh z3}b`V0)*)>7I0VxxVvj|t6{OO?J{8!inVY?5LUz-*`3#PT1g=Z0zsd>dvb{E$+v2| zZ##GUSVrdpGYRmi_0>t2P`g5_TD13;>Wf~SFvbKaQBvAk6Df>nqEpKhGagNEf8nP%Xg z!&Th5+Q3aemMuW^In`pLQlLR^V1|28HPVY+dCU*XlALxzTx_PzhWp< zqQ7kv;G`0%+2$kURdGK1z$ru!&A!a#e2!;Yo@U3 zIp|hWc8(RJ9Sa-;IHGQPvJ%;4qz_q8(TbS~y1$Z!J$6`FCxUyi3n$t{yY6QB=`19A z<>?(5k!v+OiO~a`5tT4Q+1N%Bu!q8p+Fn(qs|it@IJE$la3zwl*~e%cV}V&jh^M$^ z6ML{$#3|>->1}@tr^p>-)liKc0dScP>}7jvr-8_lQgzzwf)EIe%tu+4ma!&5z*^CW^S(I?K}zFVYZLoq zx2Wxxt@L4a9I4(SQCj3w8`a;Re!cnh?@uzc_5Pic5?25H*H8bR`BtI*=!Dbi!-IaD z3VE)wm=PN)zvb-b@}F`Rg85Gh>Z-$L)5XbrHWC32ol7#SRQFW&UT0n9f2n)E=Hc0(Ec!EK8MAEq}YcW*7 zP4+rakGu{^_ZJ^V7i;OE16F>?*GLYzRD#f)L?`#)r)RVGRsvVJLp5YnS-SDQzuV?n z!yoIn0VmH+D!ylzT@^F66x3a#l}t%z-S{uX@{3N1HdrEBJ=mnUUalQqSAQX&xS(u! z1(BjUc?z_90o=J8((3t4Po4i4#AX{29L0h-9+fVi5;5C^Xtwi)0BVW#SMOpLJSIYzzV zk%+jU_n2VeU10%vG;}*!1tDiLB0)YbSs(2~J>q4`3WBgVRl=^+fd)X-(AQn+oObkT z-`+qJTon)tkdEf*E*Y;`T=~UB__O3>! z_HEfmU#(4eKyt1KlXwVc%FLNT(sA8NTcfVRcljC+D)mi!l-Dg(J$aj0N)pLU2-_() z*~r@*;i+o6FQ?Fz=cUP=mFAg=X%wg<(-zwHxP?`|9u$^5gzx^p|)yC zY{+#?|9+Y$Ah8jQrM4WegDJw}!CDH#Uj!Kq&|faUm7BkNi8HnAEAf1NGxKJXa`0rs zA?>?dkp6VLf5N@9khq(v=4Z2*31F>=G!x0akKQHja%nPzCQ}o0wLyg4OV&-uV>Qq4 zMzwF6cQ|vf6LRlqf2|r5ML-AW-TO5S?eUZdK|PY#XD2#rNTg8Ro+@W64p6UGXbLAW zdq%PE&(I$+_VrBSFw{oi)$Q8UV-0OvFm;l#i0hjz-%nviltsLczR!fY$XB9Z000Ii zI;zo+#1HCFmsHEalp!jkZ9YjAg6Ko8DS6K<}(jH^vc> zRa7ybWlM3+B&|J0b3+jI97^ zFQlHRm8;OLrV2G~~c{4ycT0PbyMTt>i&MG(+i zY0H2-06P2*0H|cj4PYglf%~zJm|`I?C2B`+N2v3Ns6ksdCXI%oVe^TgT+nE?=VK|2 zE=e({WS}4@bFa3;Fxvo_UK>(#9CVPMLe(6kI|;dr9PQSHAUAAX+gwK4{cO)Cn8l${ zEv&7tt)m}&KE%Z^(h6z?acrc1jAT4CiE7r@#!zLDWAV4nUhS?lLKnqv7iv`}JH`g0 z5sL67kth_zB_07A^C=PNLPOp!HE@Say0l>dNNcE+TE_6{wRJTU&Kp?i`Dt2PU(*aA zktxm+yF7be%toH03uP{QbyRXtbLN3bds;(g0VZmwCC&AoMO=2au^1;aocc7QcPrdUuljW7mDA{%`wLl+4tcNy3Lo2CPE9CR8pd+V(mu^mjfjJ}e{oO#k6xTO}m#>!%eUfu)RUSur$$s3-gA==l6#faoaOBdD$6vR=LXS!Uv5 z3<-Sc06kP2p#R+s?0+{B`MwobOepzRRLH_#ztCv%qXu>Zl85GGXfv51_s~0!NiB_c z<7FmHy>!fEQ8wQ6m{bhQW`5mVhd&(sS~Xq4%5tbkU>MX5qH1Eby1n9vV21ZD#{h4I z5oFrWu!5G+>S_bDzP^tBCw$pGJ3lye_KuHUIcMj)r{~W8v2%2M?wlUHIevTa|AH2O z9Wj0q4}B^pSug2zz0@Gh^O0I@J?weJ>oYZXy$$o19WuOpZz_6(%`){o!f2+V`7=ig zNmb46&oO0c@5w@=Q?e0pg2&{H{z(RC$GFt3Yh~Oo6B#>sVH-XUb<8=c)cbNi9`W}&P4c5odjiPJkxZb~YP?Pelo=E~f5#1}HWa;-L@>Je~LY30a*RM1A)7SNtP9fQgwKvq7zg{&V_X^Es zl1Am)cF-l64N$M4k6yMsNB2m6wGz?O7A9c_5o)*%EpL?+SnZn@V7(WjNyQ~a6l0;18d||ptH_Q>nT2U zAQZS7MmW1;+aplKLC{4y@ejfy^>IkR>RKB~=a7;Fxv(!@(PR*rH*fH@Ugg{0Zf)gJ z_nMC@vFF3f1D|BB@Kq(q3%+l-=>ToNkjba@OkhYf!S>Lk!=OU!;9O-u#u)tZ%x4G@ zqnXRs+x>K|1YzWApI{K?255V$wa;9$TbYKE4hCq)w!%iTsNvuPa_wk8Do z(d5r0LX>RT!$K~i0s409+pW&ZyF+7H4B}lM{VcU1g(M+{OpEEUJ{17<-B03EnPdS< z6OOe3YVDZ^o#A{!S@aa-h~1OJY;r-pMWGO8xes1W$g?D8f_Xh5{c#`lUIe`j)Z3n@ ze|IL&Uj)52p&+;a`Wq7A-1$8fs_a1DWdyBm$BYDy0-;$0^x^G$)O{emTcU&Bx_<40&;=AO(T0eW+t-x67^`rjX{sdSwya*WHwH? zhYH^sXC=b&66OY^69Q+!ajS_WBO3;UlMyXPvBT*;by_;&R(6p1$9X6112i)WVU9P^Vj*vbh5hYct zNU5t9OBRU9h;Ftc$7V&}ii$1{@sRp7Ya#9l9uCku z)Z2aix`*D^R~7_&$IPFdavmvZT$}3T&M{TBVzw)fI8(KfviUTpcBGYRVqD5%n(DoD z6buuks{yknE(@uI^)qZkcfS!8aM+G8T-tj&NVH~L#Ve@d1nRWH>5V$1vGr_BlSI>r zbt6S2S2$r%Y2;z8mQgTV*p~VlXBWV^ zE;N|-ozA%PU;pR-+QR?!fBrwy1_phYL^8?q4~*J%c@S_2H*CFUk|VN)@6d)Dr8+>3 z7?UJwq)U>(lZ8$daR2px{;vklzR(F&$aP2rg~}Ri1nSvX5N()pspir*!M&&ak|dz0 zugAx4&`(scqK#}gRNGW0)|tt)Lh^7Vy3D@z*>y2PE>=j7S>l--WG3J_dPP*9Qu5mz4g&3akA;A?0SPN^S63n$fo5_ z1m&Y_GX9PiaS6RTc1AX_lyuBMbXMJLt^;A2W-FI$hl4{F;9EaVWm4cL$T?5nEsh${ z= zme)+_K9wSd>x}PXZ%iUJ#rkYs{E`)F?70tBw9u|VY1wS7bxO-w3?O-%$-K80=^tB3vmN%5w0la{+-6UI@iDpm_puQ;wb3N4sikVj4n~=>W17sB% zsSWnj<$RVjn+ZsqH{0D+XWkaxp^XGgz09Ff9~g1IK-S!-Ze@0zY_EE@X+(n41t>hV zc2HD)l!f{Lye&^-dnX4xOTS#)q{5p2!^%>OaoxU&1Rjm3>*z?ol#Lsin1Tls$cCzk zb_@@>FeyJkQyFnIikU_z7G!|-kIvAO7tc0hFWf*co^3{MxB=H8aD^n$lW({5yXVi% z$Cu{UQ)?}A{F9Dyc^ncZCv*h%G)b`z)0pvzcQn9*>Q!OC574)%S8*tK#9i(W(D~j; zq1f~1&pMTRaf4z%?-z@`Z0`0}Y4ng)5vh55Kd7{U6eydI7Aoe--dL)Uu6@CG9vy_6 zvYzRVk!76NES~i3;4)BTB5&RdJ-#vKS*9SoJA%)%ls3$ZB_0K^X%b{$%p&U8H;YQa zwtj;;Ms>ZLD~8~1mk2?;eaK93mH>Y^W;}U)a82Ae%7qhf&=sNMNg=^;n5en(F873csX=!T~7c=6YH<|E0}bVIBRub=`3sYu3UMT)3X>Xz=gQ)RA+3Y!tA zQxn|@mPtGk8x0w-N+n!#(T-OTx~#uXd_2oV#hv5kex6YnL!5~PH=+T7@O>R%tqR_I z12BK&SBQ@y!qneMl60`j&~c?xFgpOJO=4~bg)V0jSizLo1YfCYU*plxfQ@gLQ5tD4 z^hl???=TS4!Fg)e_bjo-cx_hGy$#=-cqEAJt<*E~7cMR70~QB^cW^-JZS+F!^)`B*Oiwh&Cc}+&{N+~+ ztfb9xOyfA?24hp=^wC@FQ!g8u_(FAZv4Ji^?p^3qJQtq4FjtXNo604`aHSYc1dqoP zbm2gkjvyncuYGG=c{S?#8twYj4K+_y#cF1o;4dkds-Hab%af1R)m3AxqpPbyZwnnh z7+2S;WRW%*3FZfOORla-ClAuqH3$7?a&u>qr|h=+q?&0ULUMXC6|7|Fb0M z-FtAPF|jj>`Di-?$Fe}G=59%TmIXV&5L9ZsciJq$62;~bMj+#ALb;sJ#?DnXj%KlSY?Z@y`xU<&EQEp-cN7QDnaXzrTLkPn&~SFKWY2pkqAS|7q-#&CK6F2p)$vVQY%oWL`}% z%T(dx$|3Yd4xxry+&72Nf>9?}0;Uz+Fs9xhJ2;K4E5#BZWUxAp+K4z>-W&SZy;R@Q zW)Q+rBHg{N(SFtY zvx9@m_vdGOC*}8NKU8#jc6LztvI6&KX9v$J-#)K?TiHZq8|VAK{{F+y2WR=gCnWWY zCB}rLw5?`_!Y=h%g++sUn<0)Sn-SkMuQSc+7sp{#^vUlX9smC0zj}q2zkd6&_-g;n zVe%?;X{H&?y&-H=8Dvv?YiDb#*y7f+;+yR(?sT(oPn(T)otAc0kHsc*1lQtQ-B$6f zjECB{<@f4Z>GwkYSPthMf2ngaD!v(&%8W*LgAX^B5f3u$Y4e_R-!R)R@*s(RByeQ> zB(jc1)(r&iNbOFWw&@@fMk1J*4k1x$4m7_Uk^Q#Ss6tfrWOc`;OA8`hTfUC$^)F>^`p zeKWwqixzA{Gv3ANLxQA#eax0#kaST^0mO4 z{1^S9&)v%fU=_yHO`X-t0B=xJgb%b8poh$pb;01UeC<=WH^f>;pFWkEC(J9n z?l=X@-M!1R#6!aRUssat+k?}y!{ejh5BK}Evh(&ch$#1>iH3_agLW31o0K~iXC-v0 zqrbq6z*<9}J~`@-q{esPbWuUhH<_M#j@?C;$nk{o+-y~|6U%`t$GI&PP!(sbU&SU5 zvdjuIXo`hRvyGXDhSs>cB%&;ffCg$Oc| z^G>I!w~YBz(Db#?xyZt4pt2?f8?>I8l`49uI#MKfG*>pu*aUOhHT3C|Y1+B{-L74) zmlHZNxH`tlM5Rpa zn4p{_l>srou<(?rg;_g^eQDt-yW?gRo^pQ6(=R4Gqu7UDPk-DyK0}-#`Rs37 z*U-|wjg|}UZ|V~qU388b9!E(-V4#YDD>)FwR(;q|oOGf%);acha_W8q@PNNvXn-NQ z5en=szrpu!2t><-19EH1>gDR)OnHlFe_G=C3Q0p9^xUu1m7?&x{!@fqhJ6dQQ4h>ND|hs8`W zC&sf9oTf|;ObGa~7))WdxucR~ywEJ+FqCSR5;^#1$c1+XmJ#D*()3#-&FiWGq}N}_ z1Rjt&S%H$GZDK>6X-^vhTm<;~XNJ?@-M#ItzxFPW#y*XK#mDlM+E>|RLo}JmyMn%Fn>x23|X}kA3-xEAhs_6U+*b?4rf! z6kE`g!B`)i>pUVCdI=n}eWIr3Vgvle;Y@g8=30}>09MBZH1Y&6<`EJ)0TE*&QGi3M zCQKEW9W<9lR@1kro0_1^UydX-d4iHP6z9lWi(h%5jSN=1;?2qPy_MCfh`2?FSx#X0 z4C81QIKS+`?gbu;9#&xY>@=?Qj3bzm`H zL(UWZMMt~AkkC5G%C)|ZFthftFxCmb(#cp%E$Gi^@qNxOwPd3!n=*wkLmt^4pm%N< zzt6tg8K8Gs!?9ue+tk*{yGgmP2k6}-9+MD{iF9_}(D!RCD6F%Ws|^WQocN01fSsqg zm?I~v?^P%~;vTV@Dq9`xo$i|$eQ$m~J6HWmt%i<7Odu5vwDgs3{Hs*U9y1USRl|Fy z`})pOM%ed>mt=(ZXFBlE1{%%~W{~z9ngcn^`VDY_$aO>n!#;{Q_n-DnW;WY!K&1q! zC%aHB#-Kf&H;4K@+!|@gzE?M1)gZ4P43-vdi^*5rU~ehxU`2NiG5}%CI6hM-!lIDynS&BmAg5D!!n?UauYhiK zmtMf;T)4aWwYs>&YxTK?rf=Bq=@okg#P{d2y=tpnbT!^Dzm5usYdk!fwBbJ{m-Uts z6#9J<8GND<^}Fvqrmj9hp5SOwdF-4M5Xs$3lOi+ZUV4zS_?Z{F6Yg2DSDtbAGKJG| z&^c9WlGU-z=NBMIUowK_d2;3jvL&JNU9oK2SiZa5+C zB}@lUjO?R>AdF@PK_=j=EHoVn?1Iu~&KW#YwfCtQpv@^3n?42G!j{bDTaZ-%Co)%^_-_(fXc-wb6-w9bDFE;)t>_5uuICd3B{Q@?{%Z=g{^5O|7e^bY*n?c#W+^5bu$_B*3$>knCt7 zX|VOMLJhSH#>N@Cdn-b{^>nAo)RfFxkm;)s>Y_0gLp)YI)7SbtBw|)`@Pfu^Vi~qQ z`4Y{WG$U-n(3fYz)Q*!1Go?pnjsnAczy>IWb36o-69ek~yLC2zJ;-}z1N-AQB&k`b z565(*X{FvTW0BQDy}gC{;7vek8P3D9$tdbi8irbJcd`A$LomCTW0;ku7274#itU|m z3$_*a!HvmC@JmZg^8jo=+NJj|#=fH+6)PK#4oLbU?KIla@@3m%j9t`CkL@Lez6g_x z4$G-7uH-^E&{}rGeD|poaj43CAA4gGDQ2uySj8_{4W~VFp^6rAW7D^5g=udP9f~n= znxrBtZX22P2^UJ9cL|H|m^?McB@#sw9O(?(LxPf+dzn+2oxv8+0?KwX0T5e&GnOI| z*Uh7tY58MjV&6;%L%0%s6v9nX)_VR+nmpm&ff<9n9NG1+@JvE@_FMJRPv&)gH+^$u zRqdExCx)I(gU-`vD3G*zj?g&HwEgfRgm3+t<8zx0f{CgmL+?e zF!_`yP0d~43HRP`Mk6kaWh0CRI1FXurHopMI`I& zh9HoH>}yhjh1H>tt*x!Cm(QQW|65yIx&OC!UT(eIe)(_PFLs_kd$IG)&NnaqZENTG z_V&wvLtA$+{(p;QgvGyY-MFvf;2y~H0hWC4FM39M1Jtu@O--(2CYp=!rYGbjbHcj~rU0n%Ss;(w``!+p?^#>$$$i59#LDf7T4e4FfnXMxv7%TcpPO;?0xi$5>Acyvyq9_2$^n7 z*l;bu=DWvk6k{L2!)#18Kn2VhikToj*jgkRymblLHwa1wYtd(Hrh3WOPieuPS&Tp6 zsDRHZe7!TNxzSh^U#6j;EdmY0fmh>f(?pj+QcKdVO#)s&+#IzeUqc*4M6i=;;z^4A z$2;u&x$FGv{a-4-=Q`4+HN8@yPixhG<+x75u6v+1C=4pft zZ%NyI({5Tuav2y)&BUGO}Ohrm9tF-O1rjuf#vQ+-9(|M3F6L*WDTI z56{-UBhG3{wVb$ihh*B#(*GGxj=nNa{D)DKZWQ-OsR}&sE|s zl^Z$SEb#zTFGtPE(r5X+oRpS>!TsGJ!fwr90uEU?qzuNC4a?8Eo z3wdw%?$<64FbKc{(O}-1`}Jd1POMa!D^=#BrQl+Oht;J>^|MkbH!9Yg+HIc zZ84N*8wyyN){@m80&=^4|h+ZK;+u#;s1$#hrA z-E|zCyVP@M5wf5K-&*xwIjh>MZ zjDGfRtvriJzC9{|u9i(C3WIaHc^oP@kgACEMM_7L(V9|(foR?!gXzE_O(NLg=!i!r zf=I&Z1S2SSM5a}eG3qR<2@-Nil?XbQR}4%Y%2ZyWk!^&>3&0ZLak^95WsUYGSVaBZ zFnmjdlHawD?U1OAwbhZOVFp`wzWIdcu8XCK`880r;V7StVSNrpRmY&RIMvsQU$ld( zxlvs?x!!_bbuI6=-?c$O+4p+6uGawSIi_rmv$W1M z3LFA~j)SuCm*o+jY%g8B!gb-5XLze;xFPw+;T;Ybn|B+}AkDAX(ak2^xT#}7^i2`* z5SbE&=m>GeRj#O1OoXX3@o8pV_^l;REhj?p->NVY_ zUbov@TXkjT^tpW~VHG}?+G-VAVHH}T)7wWLTH(&t-zvPqD!jsNhF565(=MT76=Go( zVqq0xVWlmu8Qn@-oHYGNDR>oP;eO1+N^74!pId9cP>6-21p5lpjUg7g8yVkLn1vf3 zWll{Wr&=iw79z_$%m^Z9%sm0_KbL{>?#UtSn1cI05lFXV@=){72>U z)aqz+-L~zRU>uIGT7P=aV{9y_6#0w_k)&#sTI3E4e{WGSvI=?9J>lbS_1Gew zfEan_QBS_4peIJDG@nN7Ojo+M`mp?$l{V}#pQ~2du$4CK(NgeA3zI&dTMKiq!-IV; zdyTn6gx%aFut}`0n8~&3zp|lSl<;>vB|=cIBE(c-N^L=n`iyMqPOvD&f2s3hEOrlG zPm99<)fMLi?=0U>0Ya2TypO)0A&-o3>_;1gN>n0`L_`FXX@Acji64}L@^l$T$Yr!o zd=im8bxerE)^TY6@J)|O@Do>nsos%l$?>2mG~v7y!nFQGl7mN34o7$&OL8mtbo@mWfpRVozs=hI6UW-3Z;>CJrQ&}Lb(Empk4ty`-A@})s_A}UDe%Z0DEG`^vjJk?)bt~a)?gq>EC8W)q z@vint22=`d7G=gtTJ+Hw(OYFcC>Ba=w3Zpaq&Q)o&H$R`9D9$(Z>> z>Wi+u(|xTZ0NIUriEq3=)0cr8YNy2vqH|~oNf4i4Ns!v0uOlKD_EE&S{}gngnMMOD zC1vA{94+ORpN{S|!qnG6BYx}E4N|qoc$5PATMrQ-mc8S9m;{_j@(|*4l`y&wz@L#- zpsxHqc#)0WZpf~C9k8oJGv&}-UsjN;icO|VxU6EeRp6}Fz*!~IdmJIORxzGdF`hnS zjHf$3JKklqCxt1WMEYoQf}=@i?Ek3fGQ`m&Imwu|%zZH$QU4}%cl)Mn2Hq3ySz%b7 zarbgbSdiSkG;Vbv_Yxb)#H;Bcpyn*X5!t8RJz3AQ2vC)gM+G>0&QdI0;YdlkV^abQ zIwr)u1hfFz$UZso|>3+<^O52(~pIh7dID#A9Qp_F`BPWg5V~u0ea6NFF?fd)+3GVwt?3SFpibGq) z(pgR{ozGzlxQeQCFQV$SGV^<6qh2?03+NCr%6&0bi|nxa!KK|q74pd5c-N6FrJoN zs+A#_Kpt7k^X`r|*-!w`2F&gTy7>A6hyeT9OwqUMT&%XJFWjeCqhRGpd>f8;w~5>P z;%PUh|M1bPePqy*cMV>qua@qh)(!YxyZ$~T$5JzqD|gaHI{$K=NDTtYuA^U+=V-!> zy7-Ig!YeP)RxeRQTPxqtm2c?EH}rnbm5X2dGJH%FzjE-qB?rIHr8c=Ow?2Nwh&Uyq?urwwTaY^5JeCMrNQ3<9bEcY@ zS6WS;xQwB<*r%ShRM4@+FVv_mHqb@Ly^9Ulj4nKRVPe)AlLiU8a0hJPqciGrE=9J!$%pQt-+T^?uC5>iQ#nKKJ#< z$`5trhuQ^A_sujE! zLD*ceS>lG)D`Ge0oXz znF7qmVoVb7p7yf{qf+PmfRu`=LFiW-;;M7Aq!NJ!~kCd2LHAoO0m0YRKgv!|IqjO+q=t2?K039=(9s6PfVP9&miwiTOnkWWHv5gD# z1hC@~5;_49VC#sq4nwI{U_!n>=U2#` z=o@ngZQqFYr0=yN4re35byoklV*9`y{6s(N zDl6WWuQ0Ls54EqN1YIm&9 zqX;T48jnb@?;Mf|o>DFfIL(VyceI28e0>&+@d89KA;{d)X9PH~rP2~1W{6x<8L1&9 zn@>jRr6iwdf+OT&h9-Eb&MhH~3me|>7KkyMN7=r<&2OD+C4xgAu|@Kzt~7CM7^z8m{6 zf|1}sw#(rR2_^698PxL3* z11=-=hds|mJi>lyQUKTK5naTL*|Vx4LWSxlv`_jV#>TR5aC8JMo$1pS@t&qL7EK5h zV4`{jwavNTURS8LxeqvoEZ|sNYBMqq*n+2xp_9Et$-oLvK8~5ry=U5mBY_zp45p8c znVTaR(?vQ*PfiK;%*GTUCC^6+-|+ST2LvLSJKKqQJY z6aw9{84?6b&MYvIz)WgJ4@TbDmt;fr0LNG2Z74#e5_tksrVw2oAPoZIQJt0DVmk+~ zbpps<6-l+ikxv5}$v)ckeSVd`wqbjX%o2p8v48v;t6^n#Q`$RwR+nopd8BYX)3c&e*0P|i?* zXUN6sq+^u@h~)Y zXbJZK-AA%KBN~tz&I`3+k1Y;|OH!IHrZ#ZG&`{BcRJ)qu*M(9#W`P3F$1X998#GOp zw_`z4zP*8Hq+1OIcL@RBA92J%=bUeQ14WZqBEqDlQZ-J??7z{Vb3wEXzl`znt}dW52Oo_=yg_KT}-$5(Tl0v>a)L3UIN5Sy3WX#xG7$bXo_LTSH=F zW=I?mX!%dhh!R0ES&;jv*sEGuaPS+N{zzhHv3VtEL=b0+m~Evg;6@@DhdQOo6(d4U zXo$4>3xb`f^>TLbnzHyB2?A=tE_s2Q<$UWj)_fJL;XB-~x$C{Dah*J0A zVf1o@dL;D^7O7u-bf{z|mqz{!Q8psNc*I5xL9m6?Y?@XWR=Zj{f3LW(pEf-*-IN~w|Ji%@<~DLK>stD8oC`s(y{q7scOrTIjQsUc^M|0xLSgZmH zk3=G$zkJt$&)}v^TVvvuUB7zgH}_IbWWh}_?WCiu-}F~&=+Y17`qkU(SL{$5=-so9 zLeQ0Ea;Ea?M^X3{ zqDwx$Vugc)CKbbr1hSs(7d;mnd7UFmU3Jw$Lr*%GS;Wpx0=p~gD>9#RMY6-A3I9v0 zip2Yew>$YBZgq}6;)1dYgJQ>;YCZ!Z?()>Ss}N7^2?j9&fb_< zSss~*HhrVLq`EvWYqn!BJ>mHp7%h|F#RJR2%i-ePl(Q|TXFy*&(J}maExgeF*s8@A zt<`IHz@*Bh=K-*Q^jo2M9t^4!Hqx@!R5qBcLmSXG2LdvZLyT&x3k}g8J@(1JIsrs< z|B$|x<`q(>^y)w$DCQ7!$A%{oK{%`WxvdG_1)qtR~2h z1HpJ%7N9p_9=R^jpv#STf2|mcGiTjo`${9avs>8n4K6C>U#D5srfM@xB06w;zZh;wd^;sH8Y7NM=|)Fop8s|l!Mxs zuJK8MVh4VgCn2Wt5m98TjK~Ymkl_0OHT!pdyExkozskU#H~OUW)lW>)Ht5d3_$58Y z-`Pax;ypWu3-m(poOun4{`rTI+w`eFvV0HbP%oNZXc*>}hT`U)qCoQ@iD6La0V9oN zU$gQ=vCgD^yEqGT^8W_p&In!jgOs!`fveuGb7g3jMF z4on*@(s@FghbUR7_=vnA=m)mKz3zo;5rMj4$op98%}rBj$#UcJh-HkiKlk&Tag*z9=rL0fZ?T`-2IzVZT?KC z_MtVdcVg9LsGElCH0ntBh{6?6dbdQEm&Erd`whhkPmRBdM*-Zn!K-SbT6w;qWtc89 zYWSHwLd&ZSf4f}f`0{LXr!d*K!5$~I0_$w_43V(Wi8s?=igATb7{v>Ae0j!lBX1W2 z32jxWz%T8DXHefOi=EIc&`oMdsFwy$%b2Si3T*(|8mw=R9cyL1vI_y5Q(YOx^NCc^ za-3;nMCPz#aDb1bb;8SGB+ybO%w7bq&4k%na(!Ix#KgbS=v$cw_ak`lW^wB9e&g2a z3{XVUg-2Tlu;*MjVJ1?EhAT6boHf`ymWFZIXeE`-MZX#p(vB~Iqh5e5b+vV!*9rPr zA_!BRdkbtE4`E>3pb7{@H4PJ3qO@dAo0N5(Llz#L*W*BT1YF<$`kN{nzh}HCEX9Bg zr6?ex5=~3Iz(_<0BOHX?mYxv8=w%Jj8)HD{PRz0iM!!N7NqDIPaYfW1K~VHe9*Jr9 zCT}pCVH)y97{Qm4%x%mlP9dsivAJp=eD5)7evgjdBj=19RBnkJtoaV$x&zSIge4O1 z(z;}ZqyXb5S1G?Dl0Q}QcknJx;E z1!WF4D>j=Gsw60656DI85SH{NgY9~Aae8#}>9jY=zZvVhuG8JhimrV09(v*j>%oXi zH(n#y(vAJN??vhh0x(K-N#z7bOf`4V2v{LKo}q$S}pSQ4mj(<4P_@5{N7 ziEnLP*`^9|doAhb8LgRDZY-3OWErCso4dH6uI`aIpO7Z%scGf?z;dLu@(=tn+R+Tp zwZ)R9icgv>5e;Ndk8HOrs!5d#v$?32K|jHsn?^H3AtJ}Kn;tu2$Boxk%=t-M&CB{R zVnhM{jhvsxH(yeB)-kCrNZr|PjKCffa@t3DGPVCIx=6#_GOy3@^AL-6>qXql0&YS0 z+4#Dy?+C1IflO$jNgDuU1b~+%$T`Zt| zspy(`B4={THz_*Eu1sa!e^n|kMBnKDgZlB(cj`l(uslZTkM$iEYVrCv&?w~5&$bR_ zn&4t9u`5c`2C!UD)4xl~SD-Res+>C^)`#i9iRD#hN zu=Fto71eh!?P3g0$)&OW0-O8UeI@d4iak#J)U~5hP(YBkqL{|0lMdYCc?BBwfM+*% z+~f$V%w2%(YVx%Z$O+0Dun>81XQKG(gPC+@gZJBd+kPn66xc_cAhp8!^Sp zCgBHr(F|_Hf<)a-6$9V#I?!r$CI=f-~4FoGQAylhDz z2n87|7$OgWf*Xh%n91$s4GeOg?cK{9U=K;EecXs?fVxY2gWUlZ zqDs;={P}|dF-UFUXBZ7c3MXU5&?M_W0XzA?mt!m3Ym85>)J$uFvLvd758?&wHKW)E z#-dA0k?kl^(<;S%K)TW;ptymv^;q8zGV&o4^inLgF@?{Pl^i~D z(CZzb(y9$Xege_^-3Lh1SdPpY6rY;l$c5(GT!3)2h_m_)YSURZ<<%+dHFp{=SC#)Az*uYbLa4K5iSSidzS_>dC zT19Ad+_u5Au7(st5){*!;{~yyfpB+1DE9X4+qbZ2Z~y-HzaxfqE;2cpeG%UJ;oZ9) z`^V9zkBn!)f7n3#D-C)^OJUO3cG&;8{CpmGtZhAo(5^VWX12@QAHV;;$4;aHja`ym zjR+8~QO`sDCCh=ChrkN-DLKFA$n+<2G=V0CZO& zZv-3KpN+1{PJn@CMnL`v0fS>_^Wvh;qONm|2#kXbP8bzOx%|o3XIlb zdkT@6_zmf@3f{w{K!_1m7q{WYtV8ta_K>Qq9K(~Dc#5+zS0g{n6Rd#qAO8UZG9s@s zI*D<9jB z(n4_W{Gc1MiUg3JI$>sZYoD!-bERWL4+ z_?$3cblgFmJFP5zi>oOsjZ_W}^3mn-*%`~m-0%!i5X8M#N7%vteb8fnjD;eVoDRHH zHSQ!0x>MT>QdA>QnZRhkt*d>AMS(T>N+U9n3n+^8I~udX_z|+}RNa!=#bW3jjAVWv zUl*BrVj?^D4yYCG;FW3hxOs<%;K|jL+#0y#4fJ9cU@O73=`Tq(mLq@r6yjDCq2zL6 zu6DYv>Tj6Wp^eOE1P&lvuU8kE=b^-I;mI9LlP97q^mIb7l{vd~+=*dTT#6OI4>!@a zVUYc*DEhjP+3ImcXRsq>F9;(J}gFplSuy0C@|J99~^-UaN7ry!^v2I?B1lIP; zm}BexsLk}?XE-^({O8A`|2qBHW2ZbDr)-!^T4?OU>fNE56}Ti0W82r8+~ymVuG7Y? zZm8>+eCSy-7HH0mJc9hgv4dH43GZ-4mVtCIdAU+Zh*b|L-6s@mJ<&Xz#F<0s~`X+%8(;tDN< z$|+a+g?B%)dshA>))W&jVqtddhJ?igy*F^G=YM=I(uLZ|F`~6xLy?-4l@l!84RqAR z2#g2qFKYq<5n#c>OB!1;3?jNg>pcXG87vbP!I! z8~>iH_;cC}a9~nFzGWh?YsJL9bgWWN7LZ|S#HcE`Nf#Wy2%y+_4xlppLoWy9=NzGw zn?u5YDApjPbn;=uc3e4@>pMjoT`Oq091eXyaFZ!7$jpAjZ-}Rkp%V&i?}Ri1^78d7 zFNS5@T)#@s{88b?x*Cn=uB3H-U{r!Xx+a>hh<{Z=Wmp?O*8)lGj!QSo>Kh2zgq}o| z^a(!+1;BgP`U(b12u>6`=Z!eX$wcImJ5fv-A3E?Q%+5<%M&tm727HmLX-FuCOB1jV zYy~3fuqpumnf7=uDnf%=E~;V;_TPZJDoA<2ikKLRl8r@ngG5q;z{?rUD^?hsIYwv` z#xB7(cFT*3Q;e!M0Gxtn2VTO&Xki=L0363mD`XpiDNP8nrtVvgln(gjN;ms!4tQRV z@wT4JEbHlGU#TamK{v??1c#JXULgUAlGGs>^Ly51M~*?j&bM!qc~CS^k98SrD}*IR z9tW^s94tKkjkB0BQ+MD`*if5+%yR(+%WruhbB=c-a{c<{gxl8>`Sjc66*ys9jk9~dhyf+Nu!0guk+da2&Tjf`4=(8cIJFc2e& zko16>jhrY-NFC3zfn#A1Iat0qE3_9ZB-ZNzj|`NS(}vw~Gx66D&&B}L!WGLah-&~z zW7@TjI}uR-w5o^yVjlKD`$hsQv}w9@L!toM~A|= zHMIr>$8$F$cuw;2g62TR9DPxgARrYx{rn+%t9*w5PPnJCWPY4pTH0Vy=d4dy+pN$w z1n~F;$;MnMQIPm69Wa=I5XuvM@Q%l@Qs;O!Ch39})B-G1(xY`o@JWD1@e&+##dv14 zwY57RJ7ZR0QVcS$FnNL+19bpUpW@eOSdG9|;6R-SGaPxFzuJs-EGs0W8o?JS_{8@Y zb3+u3)Y4E@9@AxjRn#3)SGlx=nP@wnH=$J=7dng-mO2l0HlTO?eMkXv zI{77y$x8;nCOCq4i`^ONyv8hT#G;SsC21cC%pm&|`l#DV()Xk5-z3P7BK6$a|H%OM*<{uqJefB*JBe`JHH zziB`fbK3DD&an_`ytI+aIhW37u&5f`%T+dhQY%c+BUudi2hm55 zec(kg;MvWU{-{UxvpR)*qsCYu{wW(*>Zaw&)+44XS5__vX!9mKB-ngpq_gHduT_U= zE3Po#35hQ4-=_5R?c48vLhAG)*H3R*;T0DOO%+rf&`x+!2-BoHztCTd&fMxGwX|4s zb@dOBX{58F7C>mFqYj8m&XJU$8(SfMZS%ajKvX z5TpAFd_%t{^WTL@;Wk0LUfr*%^O&F=@X#N>^S&a-cY+nZi_M{|x)ZrNi0^T-_|y%{ zDY1GQi+{E}I5QIefsGLvF?qoHw^2HZY1YvKlh0~L5D2$Z4m|hC<|%U!%&-EQWZkM! zQal8rVCV}5Iio7{D0|v!&Rq6*kjAFTbJ01AxB6~`B zS#>Oz55L|gb*&XZ`OtDJc6M}r^v}!Rk0D`;023fMGDYpsImJ~X5O&Q39|)}MVO{&+ z_eFw$YWQgLO?}dXX$Z*LPuP2Dw9}bhJ;}UJ4Pja(5)!TDZzuf@yE^WJ6Z!J^s^3&$ zu3udp_m{w{|I!!c#WHbT3op*Jb$WrudP-}?e{J;=4TAx|@1;Y1c5>M^hO!w@1ubx5 z^^yzM9daFo?Xk;Njpg?&E2|Ehh>4dl_a4}=Zah_(S13)7?w>kFye{VzA`tU=;R+7gZ zI{^oy@>0(1f`#*02OeT126xh8eXpyr7m$i6D>cQxV^X8E>^e~R^((;XAaY;NF0#on z^>aDa`DS59D9}PQPL9?F2svu{aQq98O949x_xP+%<21kI2~Nkc$ze|VNzOG94~9d* z6$~dD@(=`*;1=C%(%EG9_Hz8u-@d5G&ZZ0?_P!xWBXXBkfM^&{_vXf+@4SI77L@I!#kY{vL~X_|IbVZijY4!f=|Y_EWyuhovP(&8l-1%5KB-;P$acNT`aFCibX(q&!n&- zOktf`t3erj&+76r4W!gk=kHOH2R{i41syEsW;;|=4v-W}o*~d@C392otd@Jf zR>1#xSD4cfcefm%KU?t}h^c@*a7zHWNRclx}(=%~~8h z&FT#}@i9gW&LDvWoVFf4eI7j#xvW;lw!~)X6?O;anrR7=iFMlWkzn*%2<+zB4KkP{ z!0swB)POVQ`c(*;6I?~6AHwmIKs)Tb!Jlz!2rc)JUVQDD@+b7Ru9`YV6L~`$`3-y^ zmydFq?NrSEOMLV)+RD2VTZTAb%u(PNcdCc1&>3V&;4Vdk5)2iPmW9(2EpmRBz`0$7 zZXt@A@O%MI%HKq=$Z=z?04VQXPO1s3Lb!e7o%%y}A@m>6^`wGE&HK|oxr{TbZQfX` zA=vvXkpJl`Bm+}n?*;fC4f<2EXp&#Z2aOZ26k&DjjhNPD%oCoC{dA|j@@53wFlYz8 z|E{=l(mep4vu*y{z&%}-@itne+y=HfDndN?5n>4n$bPsKnGtSLM(36pX%;@cj(&I5 zD4!7v2t$V1KD@N;((vn95sVnh`?V!~21)Rcpe#zE@`z4CCIqObG!Hkq$3CqKg1i?t z4pg`*+t>AL8JD|Czp(_@3v#>ebVp(m6z()1P!19A%l;!|4I!qxUeSDgIgNy#3c6!Zo+% zv>wVrtVz)yVp9@$6XGTg2}aT$y9g$6|Fc&V!Eiqm+*L-f5d>La^+)B|=pNiuH1^Hb z?pgQ>=}%FP#l&CoT{3Mny@;K*3$f`|oN4`xJdBU(d)s5^*dLQ=ZfE3Nd(s4#qv|bR z`GGvvD9x$ZY-@DBI58=;;buCoROf_G8P8EtRw!9|Nw%-iG+J3nV`1JQ(l8cpg7P?b zSkK817BX-Uh+Uf$=?Kb6RRd872a^@IGIOC3h(BU>*ev6ft;Jq|R(+|g`chf-rLyY3Yh~4!x}`65OPAP!Sn4xKhm<9YBGeo; z`fIN~T4VD+jn(*)g+Aw*dKQ=bgivsZ#R)lLghRzAT8&!qb)C7TlGa+{FIW4s56+{l z(aMv(IG=n`%CG3qzEpU8_7xuMU09kT=Uc2RE0`_O=n=at{9&0!`05x1S?WD{!JbG3 z$A$C_7|M`95%P#;$~uZ>h3cr>LCp6*bQSDZQ1_xP*r?~aEB13sJ(KctyIZ^`&5fRN zxw1Odapl5gbA`bDmN5G_C^q&ri7+oksOm|+6rp-4LiJLFYGEhbjD(Aq=vw%@>AjPJ ziPWnC3*fJnQe+8e*u0Ak2Sjz7=DiT>ELM?m#84~C4$u0>9o9cP=`i84*Ep1BLH7Vk z1A-CdEYAB-45&%EpREni*EvRxLO{*jLk`9mIVlkeFD4qtOiqf2B>EDuT`U|OTGWt9 zxJu+(RcWTa9wybr(<0|}PUIJ=O7&9Q?4`Kb7RczOxY!tHu&fYw$!gEWZ|VM+SgYVEt>kbfix5k0Kr3ndQ+E&Kvi97GfB`~awBgyPHkT{ zE*-~pTyQzL5|h%)B&*+b327tL{{GR~X zR_YXg-isItR0jA4nc1*-7=m8;*&PC_zUN*C?J%b`E3_KD2A%aNP2GT%fW_jdZbowU z-TL*9SGFGqjO}kqyT3)^sOwx|fbgxBA##~3Rw+RIj5y-Hvx&5h--sT&blkWUWEoda zq4dzO(A53-A=kMyXs-gLi01_J3eFVUL}|kuL&VA4-M9AC>UJJ~z`28+ZvCGktJS!>YhMp$S(NbYMiLm35{< zNOsN!M&Ag1rna~1<6dM(t3mG9O_Bm6oL#>nD^|QquU}CqoxNqZ zua_TjjO_x0qi#Ui!{jqalweZX;A%<|%lw}+5=-Jj#=d}%wRY_A;LXA78L6Wst6n87 z{dtpM8E2^9b6(0+kNmoo+#*2bM#RHA}l(I%n@1+X{0X<}I(Qc0%i+ip5RsgX; zPQSbv3tqT!ILNB|moOU5dGzFqSJT)q7ht<*^TYARG9CLM2S%`C^1%cFX5OzBfKh?b zdZtRYM2-yj&7F}>z~!*0_0z?i9NH`FZ#A}n%>q}b&nKcz2F){t0B*k%6UxKg4Ao$< zM3Ltg;k=*T_t^C-aqqsre$`>uukMEy|Dl}!SC5@d;`x&*DX}SHGcQI}Dk?~OF*Fs` z>VDax0&{b13;%FT4glsKVR_i62*c}7!Dm&hZdwZso6}B>gI}6Z(JVd;-Vv!-qT4SY0kZ{n+aG_NuWrJrn=T@TJ8g>m&fjd{C}M~H<=4;rdU(io zawqfE*AZGZ*N=1&SU&~}h_#G{QW(n)p~Y)<)&SDgW94#T!KgsY4v&m4)WuXD7CJfNx1P{T%&!&>!!u0k1!QHnOpe>aB`Y;d-enDSTosmmM1j?)6$@KtX1^wv5RB%Ic@pYEX4gYh9iKe zjh;~PgcuD*_J(~No#(IQrISFC^4Gk}e;~VQD_fn2C zJX$+r8<38cp5BA!Exyh3U>5B!cTQ%$vpfB7%jSjr`I{Hk$FYwsn`_eedB*oeu!l%V z>4j;9vJ%9GR%O+a+vsV+dx1`+4TY@G4~7A0a>$f!eI*4Jh+^y9fmk-tMqtdOR^g=% z#6;r1F{H(hDM{jMj%xS0_B!JQqPkv$_1&Y;ptBpXdBG({9W^AVn!AUmd<5tbpB0Le zU^mwYH<5G4tpLDXtgwcE|MuJ2NfMT6h3mDxWm&EpLDF+=>Dqk35_|R3R+k=lmQxhP@DGz|+*0d!_I5JS&x;pBtPq|>|@lGOYspbdn;4kG@X^@QsCq&oA! z0{-%>f#aqUAZm}rPcP?qLwwN?3^Czq3h6+$N8Nl07u=F{zw`5*A(d_{_qXNwrv4nU zdhaOOhV+3eS-xkd%2r0yHM`nfNxOn}00B+*{k9!}n-&clWulA+%M<-x093ef^1+~# zMf)0C1{Rw9pqAO#vmw(H>0AghO=`8pVBSio>q3`UxbW@uD|V<2^zJ!GA^0tYZsFOPqUibh%hAAI zi;CxKNhR1UxlbF>lqbzw!?p@qbxDCP^&+)`8tl*t5s&9cx^Xq=WqR^vWXcTxYf$LHn+dm07=L0Z_^%6yFkbMS4Fpm@yr=5L_*uFOQR>YVn^rPL%;&6g0n#i^&PLKPBT8*0X3zXgf3IR? znA0k}K1Yf0`*?ENzc@WUx;j01&uHqnSwa7X5YsxLg16w3l7V+f&f=;r*b50>T(L7H z>lA-nldRAIa|Q;Cj+>DTeqQVhoaK$)<~UvV!YS*$*^zfAgdwkPf`|dkp^!61W7WSC zoNNe|58l%`{!->e>4V-u(1MD1riq^GPSd=YHu1900bjM_Rht%j2+>Zm&JCZ_OH|FP zLHbcTJR^B46i#eEl2+XP)n%=EMr0tm__&`jhf->_hN0|jnX=^HfUfAhFQ2bYCCKx%tQu&spYiUT`%dr z71>{|uKGUkpBsXMF_DER7!UK{1V1PAeR{hQ6*}d9ldT6MR3pBguDH&)2K0(-StP zVK-}{5Q+4VXsCjZ5hZ4#QJd!r;oxV5>$Zz1l(p`a020Iz^@n`4u>4k-2e1KRhugwj zUlX3*wCqG6P0dUw@&7GpIzktBc+2aqN9RZXy!`zb%02BY9Am<>u^;EOSKf?zm6Ju! zOEb>&&XoIXE;m?Yn~lX9Hug(smPQ8>|03sV@QXDsP)|WmRTD^#;VSnXHff|yXDKsF z)A=RcdnI7G?ya2UL($2+sCMMVU8}gnl_wiqH;gDm;{?P<1d~F>l~yw2h5Sq8tSYr4 zs4vKr&bSp_h|4GG-X;~}{6*ok&V%0`9Q*ja{Z;wjd{MBQCwh#ubw`Edi>YIt6a+0O zeAI8ugqKj7Zy?;A5Q?4kA=trl6x>r#*}}$R6+5apq7bO0S?$<2wEN4flI8BCOql;# zq9(3fu+i{=z7?>rd2WRLast%uBXMQ9Iv`_Qmmoqwlm{drArQ_JK7U?(+uWW76GnhF zUX}rbluTE@#D~wrbhSIH)cS^-5Cs_R*Cdvr+(fa%!NkLvolXj=ZU8NW>>FMcBIBaE z31XKKyjX1fS;g<}2l7bzC)?K5=++7Of{-lF&;r#{C_+Q%ync24>dCFdb`Sm8{+?HN=1N9G&e`ckvq zNp9OqoNI$=OPg7_moleHI?0`0uRcMeb@ZyD4FQSuEW82cClC#08r3dW@Mgf=&y*8p z$TQIb^yn}p?x=4W-$9f}3J)lOH)5j3`vv%swMI^$U`drNFa>SX?fVFqUU|{~rCI?I zxE%jo@dB6ed|%N6&8=E}Ce59uN!8nW9zC2;F@h_asWV(zzs*e(-2wR?4%nf~%GXaC zbt*{+$2nc$R(_1IQ%^G156{->Iy|+DTbxcAWf}HSS{J}<;Rk7%KwuyaRupZREdS|M z+xK`ICiZb@vGOtIB!)1(LpLpD21uV*$9;IyvY&o@3-a4{-~RY^c^Y|smV>7gL;ai* zbp0Gbu)4&=SEw$4Ob4kIP%z`Awwn00@08Yubk2UDee0;#tV&<1{R^*2+O4&{F_;;|gsqaBG3M~~IfxdBy(SQ?i_G*ngegY98e{^pmksF;7rHhZsg ztx-NJB7b+1w2zd14-(}MP)nlAOjykZIFkqrhD zQJ&HzOF2I7l<(zz!JV*i|3foA=z2j5HEtsOAx>*ge^}qi1zk5{hpP$Y%yNg4!b;}7t7?6SSYZh0&H_$SXsfXXZZ$tv#g1$9%03t z#xNN+mLq=+6yjDCR3jrb!9fjQyd1QtORu)34@-Zi%}t?s4iAkk3SlfO&lT{`Z>uqrOa3b-PBzEFM@(<}X8$+tG$e&Qt<;bEMB%7)I61ip+v0*h^8CSk{?5y4cE0!moy(;&c#&JhT!_-F@ib+m*nVg~? z`$Uk3NM?Lsby2w(`aK5^5jquA>JyWC!fQ#k58R8$>9z}$xLq#(Q4U2m&5GsKf!HDq z*vb+D#o&{8L6S+FkZ2sg0WvqjmRi{mwi{|1#=Zek*7g8805j68gX%Fr+kGQ0oi3C2 zu@;lDorbTxU8NSv%-E#LOh^&7EYp*+5KeShr3ztft0h(aMNzX;*Eq^Jm1(2_fS-8) zHvfI;Kw)LpP=Ugl`uXz$B)Z8&W znoJds%o;`W@u&);D~d@j{0XI4BcPV$-W|dbSR`-$XlU=wQqnLsuDvz zL&?;R*8shuEw$>@Fcg&P3qJ~%q17ijlEA}h_-!i647Xz}W-9JwC`^qhuxUxvk>Q!} z!iv;hgs+s#^_{<1M^f3a62Ply$$%QA4W)AO9=i}DZt@}tiPDwJbe(WcLvEiVkQrc^ zuwGPL;1u@vkr5vfU&}?or|6M{O2+apVeIQI95=s!^|OWJz~WWecn$C5*L*&$)8(Zc ztpx`jnqwFF-V-2^wma=`B*5boiq^Ol6JZ*iN8O9JkSIjaC57oOXY>!9!tiT z#=r!uy1veB$aPm*brBg7jS|}S9*MtZPeFJJJHN6FMGfP=@vz=m5KX9>RDkjG@)LkI z@l^QH`er`pA_~5u$ZaPe=U*N7FLib!TwT24vHCb$mRLMLT$#vE??slvyF_r@MlG^d zPRh>Deo`oItyAy0-p^uY~tNuLg49Dyk?#DOxNpVyI+ZBy5o8*un=N(0p-%S zYPuxy%JBN|G?^0}CHU6vR|N*Q$(p#V#M8R+x}2mCBk{~j%g#SvC4ouYFW3w}@(1h2 za|_R)2RysE<0glzo;x{^g>+L8lMEFkP0N|;VTxysH3L&lWfLnuck#AJZldK2c5S>t zab+Ieq%B_^^E{|aUmtTC*O1s4?<1!ZlBiX2CeZ#ksN9g6cm?PNqA>E#ukWPB+Oj*ZOCTt{k zi1`bpWKo7eBP5&>?*G4-XzN%ejNhnY%I=I-qm`v#eJjkZ6nAf^z-HHfv`dc5H?U=I zzQGiirztj_f4JO2p!NBO%cW65{JYO1$pUkOzl>XAeYo7M*JkUm89Q3T4P&6pI)zlQSuo;_4a zL!Ar8sTHO@jf}2JzjCVa%w}VeR}lJQszPjvAF?!?flJzm9Lu*|#!;M&l#V~2f=;(i z7Q>a7Ua-?(I>S$bzmXpJdjU?CUlMX$Sni}H=-RkQ;@)36=$xzr#0S$v#%+{%&&FCq z!Dv`(==WA?h(ST6++fj)P_3};9_@~GGie#f3vs=|$?)COIBj5$xBfz#_)hPH9F3jr zFxeA55Z?vQ#!0sm0cA0YB7$5dWPo=R1|l3r>niD!ohUlBcW&--p(XZp2P7)5o^u%E zV9KNm2@q~eVXLtQ7)+ebZ!JrE>oN$+oMR492^fEzPcyd~g z?*EF74(;oDIU(AER+E!Gx48??7bT@1rciI(l`xat1pUq@lr0@`@;ybWYtZe2QwrPZ zCBKPqUhhoE-b;&}h!WH&L`du+XF6bGdXkCWuhmA732q~B7L7tr{8y!Nq41y|f(R?a z4n=R&OS~vmnR6#z<7mmX>e&u>RL}|S41|Xy?gDWuvsQgh=`7A<=Nh)Zk7A|Qecqfa zmzO<49%4gz*~F&g*RE30x)CBPfLEnl9drfuuSyu{`TZ^UPpxp_@&&d7yd83oTVMEm zQ3DU-7;ODYEmb#1>4cf=2l`7Ti|QGN-4W`^%xBgOknFJ|R!~{ld5Z=YF8mXi!{>|V z%-zph%l1TSJ*UIEsF~yjhcfm-IsI`@0rfao!9old)P}~0D6S~=GQ13NLZS|;^Dkk$ zrx1S_#`<((5dCpM4hO+g!MGlGfCHYf7pwPFf{_+zb^eez3vR^pz*5MVR$vJNA1QHd zq6hXn6ul0gM22yuI_z9K|39bq(t4@;L<@Va9sJp2KRcwnkNb(z#Idft97l=)H}lh_ zkhL&t5_96Qk(QkSt~E^?VjM$)gzqBw6r&?|ZXdW>zMpix9_VgyS3iTb?&HG7V}1*o zhg72?>KGkzkk{x(PMFe&8UTsn9m$T2{Xygc)B%W9q9K}@J28?On+P+4=)i3J+|&7X zcLl)Bv;yz1yq;>vo<4hrM}<%6I(6rY0579FjzSeGbVuhW7W6%RB-B`_0j}|ZMK=G85kVJ;9Xrrkj1Wd1w%Pr zcICk*Yx61esKQRJtUJzWOS@?)`0;mPu$DR*1N0CC(!oPH99tDP;j{gnd5?H0X^JbK z0&15UBqtpH`5}ziU%q!nI)~9usI=)}6X*H*)$P0MSNkcf9v5>hVBj<~3)5f<^uM=i zKOFVX*e$NkQ_Uc#3yh*u_qgD4a#=nQ;x)4(i+V^h3__UhtS%ha4L&pcTo*pqefP|z-3N|&&K-XwN}+ytws3KvKeYa07FNuBqtcg% zNbsKhjU6ehwCfcM3Xo8|v|}CXo*!EJp+|cYiX}8(f8FY=|NAjJoO{Y%$JFysfl3%i zr1PzvFc7mFvsGaVA#kF}sGuq}x{GRbMa&gp9I%pzAY=Ig*Q(I9u)iMrV=VCfBQ?dr zpCpSbxT}m{BmXIBnNWOCh`hs+4T#@~a7((BxDEAN7h>3nmC55z%eBSmwMN5U44#@P z;uaOz#w*Q8>sD_Kuf)6&>>*TcT8eyo$RCaAZE(8$;?!ZVU>?B30)7tE^PP_M;Zo)Z zUO(Rx6wykf&*=q-a<^zo#!O8<#YDkr<11-Xe`TY?{ZMJeGH~|7yRR`tb?!k7f?%Na zG^)2dU$U=S~!b|KEp>lkHurIir%` zg|;l0HUxK);qWjw8e!A@R=u7xDzF@zTY(`I(aWH6HWEr0@F)1&(->`#tF*tur1c%u zx53;orYnbVF7sC^yhBfY&r;!{1?usuaQI)s@z?A(i*t(yIA*smb-hvuEZ(p^c6d~j zM2U^-=7<63a?0xJ18lE~*Etu|tUqJ;{935;O=eS*v?jkuM%vH^;r zqJ$&sh;0$=)L z#AP3|Q!S#cXlVo=Y8|#P#h`@{EREUuRzl$nf7hbmv8O6Myp*;#5spK~gdgJ)fg}&- zSb7pTE;zEC)mN?6!MvK$m&rRZU@j&}Zku#7|C!}A@Qu8S@X4cSHCcZd7xon7>l7Wv zFWc|)8sJ+Y!irY*Y^E44z_A;78)898Dbwf+67m>Q ztAZHiSXr4;16qXFqFa;%N;6pqvN2a9krVzxB%22M!coXEA&0FtOvwOy;}2;@kqz82 zr57w<1oXl>#6e=Lm$Lr0P!4Vd#m>}FaD>J~A;+_52%oFt+;ATL%5nkSp>-V`N|hBA ztf7*{YhW~7Wul;n%iK$e;E|65Nv@s*`#R`q?YZSzpw=ydYgsOm6Jb?pV*@^_#ZJ!Bthgmr1bNef`zpekv{kfiXaBGp0;`Tmw?17?`sCPBMzVEF;U zLyLSt^^F2OuGVw%5?qN%S#Y;}2H1nW_~i>1M`?6kWwR^RP!0k9kQ+y?xYKehCc^L{ zO-|cS(pM?6-8kjE@Z=S+m8tR*pGMFtj0K<&I;9;?z@A!0Q5mA7`@bLK!9-|JHDc2B z*tsTSM`G*=N`~(~^kxKABcm{zI=lEwNS>$$eiDwaDRw^O3|q7TSGDC1@4<+yIhUp} zqD3etO*D5StQG7_rGik=*df@$k#M%IKVbBP@>%IvVb5Si3cF?zfMWjP!acQ=5)SEb zyI0p2J0ZU)gIg=j_0gW{i-7{utuO}`HZ%oft&_J=9;JJBC>2!*9Oz2rtp7XO1pzjl z5|-#qA~;B)J(|DWjd4laVZvI(bSWR=w zFi}{+4(nWph49*2<;QT7!ezb_rd)Twa)F-EvaNV7I*fDaA(ZYcbBi01q{DrrNaE| zWH(25CpD+)x>6DEcIKVsD@paXT|;Uwb;|vYtqeVopPhs?Jv*UG1Q5l+75pesK2=po z=PM_D_jSG~=Nn?%f!Dtze!nDsf31n%V=QG5kPjEeRf;V2amya6c#e63Mb1tb&4>a3rygmtmf7k^I!A4r*+n#Ke&!8Sa*n3 zd5(nf+!2rx`0*;T$CEo`+d5o36D?Ls#5$?rNih=&SnZ+LC<{;8{^A3c8{$BU)LC2O z@zdEE$=fu#>CnD@9kNNK#d0qZ@HEq1gsjWiVK`L3atHx4jXo0G zT5g%ClmhRW3itLM%;h$C9O7K?YPb>8P~($w$9@%4s^nQ;Ih-uYl(41)Ry>~DnBiY9 zq>LlkOXcQz=(|yp8k@gPN>n=|KBY*JA z6k6X@Wh|kB#%6yhWO@>AJ#;s8h^^d+X%~-Ua6|Y@NtjVUPy4e$Zh!h?{U;R5QS1;J zzWy_r0!q%L8pQzlL~>kecF{DN4*URF7cipNQ6%UwSYO}@xmE|{kPeo)IWw=bl2J;+ zr3}E^A%rjq;6(Et6$lz?Gmv>M6apjDry#PmNim4bnyy`s9Tir0+V5rH^a&1faOyPK zdki^!?rF~toi=GVJ#g(FzQQx7w0k*rGskXb;}@U2nO9#9AHf2Hq_|C?A`!ysNa-5X z1keQ%K$D0S1jn`+Fx5bYN;*U0pVdG^+`kv8y&_ddmnDti-5w$!)CemoZ(}0VEg+SA zBDk{b`c)96y*w0Nxqg)fj)guC|DXTwbrg)eGL{66KTpx9p1{h4lI~toY`{A1WL78Z zM^_sBYr9mFQp9|kN$G0lZ1f%>kVrx^Egz$uub083D$(jz=dJ3j!7O7RQmAM0#OqDo ziO!p>&oX+hnl^FlMBdhOd+Zd5GwqnNBq<)~$p*6N{75;uo`*&`xrCdh@z#s4<^f+# zdVF3SHp@9|7OmM9y*9QQIKwm7a@|DLWncG1?$?=j&Y2+<;H8AS`?EUceKcRRHH49QG5MK;e+E2i| z94TSWAFd4e>PAczeugSagR0`jrYOEfg3}!J60V zT7B|Y8|ao9Mdp(xQdz+=BXW$#{jW;ph3G?BYvc|X`*|a#07>Qdc;2KsY^p1het;<< zP%g{HBD<-z-#{vb;KRm@I0mIt@KuKmM8^H-u-sRvE#{Ls4Yg3}AUQUXdDl?J!CnH< z=B?Kz+ZwxL1L$x0ZM~Y?9Bsx3W{MTf8>q*T&G%0~QN30aR)^A9SO1Fr^e@hSKe{?) zzn=aBf9`~y5Zko>>+$6`Kc$=?j?9P&y+&1G{T3geUR<4hI6L-z;?Io^!SHHrCGt7s zUJHkCw%H*N5$2ps@w%b1!}c|d0zn7n?H4P&+zeEUHE~ng&9F?wO#%wbou0@nkRISd zJ0ry{m6;4=8_9cS@IB_mh^el^P9o3;n7>PI{ChyobB+=W2L%;@rXil^5)o92Y5Rc9 z9IQKT!;+Fz<2Pv^2Zh+i24@47^M%QM0VWqN#&wkTkhyUqrX`n#{aOt~At(_nK%g{I zNBs67dUix-7VP@f#pTg=Z~y)+Mkilf9)17zKYzqOPLEG6kNV&K(D?JapT1B3{NdYo z{!hOV*RM`bzWw2ccmKI3=*J1aKfNy{x}+{erqvq2uk$(yR)Yz@2Y8QI$PUVaIY`f< zbxr6F);H_|q)@3$z$mPo_134SvUyZIlh+J7+_taVQ;j zQw>Cymy*hM0RdP6<`t*ePd|SD_U+-f-?Oo{ zsB?CFd68fqU_H5TaY{>z`GoDxC+C;PS`FoB6+JIDiplq$2b`Q=GB24-1i(6kN|}(? zN%9A-i?$+JK@ieT&M*7Akl8{xV+TcH@c?XsukB|H*>c2x>XL#Y0CpsB7((T>NHyS= z9CSE}#S6ktDik1sONJWMdCe1j`vvVGyZ5ANSo%#j$_T3qzf06zYlG5Il(<3k>T;VS zl!FRQ?Tuj<{>x~riycC=s#NSV{eY;8x55}KmQ0{7AfE~zy>l?CM-R))-aJJcSY#r& z1iK&{E4azV>r^jkP!4v(ayT~*?HgHP4>B5 z)o$5p8gI3EY@@CK{nX2sOOn~=%z}b2JZqw=W_v3ov^tP~5XT>w3?jEaT`BAHB$>i3PrX`VeA`IDX%L*wSYH+LJ!_d zxOKw3$$7xG%vnsM(mKCAc*Mj0<%}aK|cT zM5aed{#7qTDq7lP;hY}lVyLV0R6Y*svIpkkrc;?-(c%DsStgpcje zx77H=sweL_ZH>!iajjy%iRCT;o<~^5k%0n`A}Sh{ zlK-kiF-_qP5TzvTPeSN?u=(`>2?5y~;X;bKa0m&%3swayI)Jt%NgVliz0&3{J%ATd z3|7`&e85k_@&OUw-6JZq&bsr>st1-i*^sQ2t-ZVp;`kE01#o=EKvGs za_01q&3??^jB}3{qQ-`RlNGWcwziz7@|lesTyw-fL_+LcRjBzF*d~T8ErWDhJL9 zh_u%P=bKoscpfvMn*mqNbqXM;h+o6Xf-6Coz!6k&W-?FqIBosruLBZS6)u}CAr!(| zok<`l1OV3Xk5R=<4l#9gjN!lS#Wx#+JOXscQ=X06zcxzmHVz;pH9cXX5?N)Wn|e`6 z+%I3He6}eY>mmns*e%4ZD43<^DYiJMrdZW3>Yf2%&i2@)PCz#UtR}&(0myM^)3eQw z_ZU`A4636ZE`ArXEq$+<$U)YF?6_g8`RoMuWK-j1d1NNq^o<_MVs+YWwGa2|5&axZ zg-$~#Ny2nVRF*2a=gzeg2~s!vz8Tt|jjqZy zOz~g};RRT6$lYF0j!W>6P-+~+3pbY;ZDmOH0T>j@1cARF_9etQxNxnaP3g{eF_tAV zsT#o)gR>*5C<){5{T=xZMGk!jp=m##oz&UKW5GUL!V0i&DbrWDk<8fuLwWozA9^VX z!*>F7F3DiAlmPY?KTch~X5W#pBg{lX>V@wY5BTSc19EGbyRNzGcDr4Qx~P;DRO7r6 zaRYJifJNV03C~*UPuP&D6+k|BO~{*s3;g)~Mr3;ogn=%aHIN!-`iKVJff(~!>3PNw zsSncV7OZ%SV46z<5dH1= zuFvX1YZp)tu3u5C)=_cCr-(#3Y*t0=Bu@jD`~C=ZBYm(^M{BZ7&j~#9ahjP58HiBM zOfiWh6g-NuL1}a*tnIPitbk-n2JO1sK0y zv)4ZIAT+r*O47BPd$Gc%as?`1b+ocDUnd&^YB<+G{;Ow2HA}ZNcORr`PbN@UL}}GE zT06l1#zZDli8*%!q37exCMSFZQRsnqoMuAV%7k1-SC>wqPyKdw!rrn&|E6Do8D%ibNY3_$ltwzYUwQ;-gg6{YKKxoIawoGbWKcF5`h^m#6SP zB`r1~Eub7Gm5r+_bCGm&<{-(pA#@~3-ML0lfJB<`8EtCWC#mSvdTp-s0jTU4A`(T9 z3r{*$-iqs2U?yw)Lf{uhz}!HxgW8%TC$=E{nC29#chXvfocwV}vrdOa8g=}+K|DYbg!quAM$GSuPxKNC z)MKY%W?@#Cs@io%2=+k^jPRtcL;)Sh%j3(ldPgAFs3=(SBSB*%CFDHMsK8GcrW2@f zP}c?kHbQ3$cHuo-&7>hp<2yXT{-FG_zh2m9@GYhT=O7F-Bf79;g~*u+1Y1mdGgKaf z;X_Fs&>?`a(O}{IMHSOSfPX_MhgDgob!TQcQG{Q;&s^GvJB1T#$|^fFT?;4pZaIG;6vx z9?jw19h~2tv?8RU=KvwbM#3@cs_e#(D*AzsgBt+~-8~Ooi>Cc~%{XH(CWXDsf(W5o zMG#wiL~?AitmF*a4}L8~-C>*=u5vw@`C0_}NaT|jL5g;3WBVtqb|8;6Ni56sq+HkH zj0QvK;%l=mUp7}}z12-j!(cPnB;*@R%QMJonCwg{4yZ#?wa#2;Hr1sm>Xt@MxS4`+ z26SFBvmMH$!t852mA>ue%(O_Jo~^J=C?5+Oz_#bY2{WN;nF5PujfE8?(lo}dQH~Hi z4`yC8xlhDofY!h{Wo)Cu?p3^59;ebwWQ2qp6aI^(QR04BSb zds-XqLw+rF`b=c6sm)xmI={ZecHk0+MAR0FX3nVRTDFgIbA1E`c4N<;Vr+PL|8S8K z{+dnXXzZe?vf!1<#?O~^Y<_jaitU1b(9&!ok^g~R#Db?W24YpIT*nkWHwKS#!+O<{ zc+-Hf-oDMrYf~cRLhN4Dqb+I8<8irq`yn|GRnMIlhf|6S;cb#<`%T5EsdP6%Ts(8~ zR-6c)7gC9JAt-P!dBO6Ku_9H(Ctp6UL)f%ob_y!5DJURfPIU1}@g+ayUnNdrB93?g>X#+(gVi>+avMiOLK zLqaFhh%@5dHRJH znS(cLq-Gc$cQOOCCFpvE1j~(FeF>9Tbs0XA0F~Gg2BOdykV3j_*~<8!o25>)8;~l; zpgQ`H$a|EG( z0JfvY`t^^s7y^6Q!b}JixMBdk@3NVp zmqid5H28?O4i6Hff-8-F4F827BzqAvNvjy+SnssC0XQ^l;51RxG+iWdAFBvEy?EMt z(MnMXryJp;R5z%nR1{hz%VMj+ zV};dc$l=M|yr(lHwu>v+bGM^Ha=Qk^%rA9Y&@2Ns5HF%F%Xg2C=d(UJa$`NI21}?n z6q{gDLL%`$-;IpRw5$C)>mRSYcfZjg^6d{SlVw)h9p|uDNF3ya-4lCF?%cJgk|}H) zNk*-1ll=@3OAdms*W(y)fsg1MD>NFJ$KnEGW<0f?VXELonA>#%D zQ)})DFZ8u>e}?oX$h9<%B_xJw2;7qF&&c$kRfhtAR_)QtXcR1svhdvAdtZ&WG*1RA(^|6V8WE5<<^mnMdM45A~0ui-X5x&*>5?sW?s(@p4M`aZ- zR~aCJdGrin&c-d51*dSfgB?>Q6R0Uy=9v}7pdHIjae}4LW^4rH# zI$Sg{il@eOZQr|syD20iHME#Zic5nwx<#%52;+2v7G@@qO`(>Q;>TE$}N1dqN?6HW(NEay8OnAfT-r zbIPLxA&i2w#u`eIVC)@2EyC4{DgkuoN$)&X$GQyuH*&k}?F6iqciU9{LpQ;X+SmCU zw`WtnfcFEL6XkHS%yxsPe#?klORU6a0#97)&PzK;i}b0emDb>l$uUS*gj=sLWJ2CH z5(67#2LxstJJ9Y$<#kXp1`k5AIO3jV+T=DxMu^lL_d#jZrDBY*sP9>qoz02FiuEc3 zP%`?d{tP4e6Nd7qtpcO8OlEt3;y-+;_zVxf2xP&yi)i1)cRtJlekOajDY7fxMbFo1vBF!I=vMQ)F zSghLzLr&xPv6lKJseM6))gJGf>Kdu70et-?*O`5jX_bl6!9N0Q8;<)M5?7@fb>SEPVH#mqt1j=BI)Aom)<>Q3;xo7@ zFT#fJLgVsd*n-sujWLD%D-^tV#Ak7)q-%%~MoN4HIzeZKh5>Mq|%7&8ai6yo9NLhVT$cap1q9f>!KYg7LR~3{~ zDvnTR`D4jas^yGT712N4TxQ#2AN_YIfUyoKk*$>l&z6-FS#VD&b_}U6?e$;Il{V`e zTa~Z*Yt`!q_D!jC`^FEpoAA14%fNPO=Tkag4~pxY8YY&187kURP0kwLDwY!jHUlzT zr||;!RFY%bt*7qhiD+6+5}7Cdcn*-ip8@8wEj%Lkx}g-`XzYPO4n(FW0$5Y&`I=XQ zf)HFpRuM{ewq;(j_5`}68=Q)SVnmHJ4XK*MskFSJKSCjn3u_TBL+teeb)l}I)anhl zR*savth>IWH{baGRu5NwW80G2{I}MwH-DZqrkY*hFRF1$j34Y1t@RvAS665D+WTxv zv|HgD2NtuE>|^_appVe!>JnD%AfDNlWq0Gj(5m=$9d+JcE*xs+J3Fjyutw}_LxcQ$ zIg^TQ6EIc+zriZdc`CKkAW&ZDFcKXKbc~TLg#11FPFvlCxIttcLmezQ)Nd_@SmEBY zZW~Uj8RGfycj(>D<6l7!8CgBkKhOHd>7QUd3_k=3A^y-FYa8%s!E4Ptvc+$u2``J@ z3b`Qs{>nR+zEi`RW0))Q}lLu7rR{PD)GYISpe)G_Ojj z7)OBWAqDqbV7kI_!L6MIfuab-QkR$#!?MsDb|1V+mTmft(9gw-yZ-g3i zm7)Iq=0=qEjg=!5wtoESCy>0tlDxS%Jv#Yx+M6tG(Au>aw(i-U@NDc^%d}VCjNq`x z*p~mUxN;AMAhWe)y3z=*ze%muj34Z`ItUPx{W_j7d;=qUD4(g&cfpV)-P9|4pqQPD zTVXnkdj05M>c@A|Ah3CF)DU3%P?5se*UI?`je`f`cW$JvY)Bi3z|NN9pdx1k4T%!& zVkab3*I?9fMUEo>IiAnrBzT5z!>5eJGEV`_qy927x4HZ*aA=zj6uJrH0A7Jw{Z+7o zB~G1AZ5=a>nLWwEjAeUL8|FC31obSq+3JIF{fZoxo&k0sQUP9T=w7YabxbcM~8#k+ zCG)pXP8vZW`@seo8w3zMH+=`;!w#EV4xk2P&j_Y>q>Nd4kdmXX` zRM;e&szrm8n;+?r^d5wM?y%451D8c*LQy(h@&Xn|O-@=&XTMxnTdS*2NG2>eGV#UQRr zD4$d3Kn1@cMui49BMP+5uH&Huf0n3A9_1`U+e#x;m&{M5QcY~(=zkUL`c)xDJoC({ z@uj;4;!v!~#uY>(LC|1EFGA6O>@x`-kKlYb$RKhDoEMA{F&h|me)#E!Zm2p5L)iE!Z~->k$LKx&uS&$&J>p%6 z!97ef(O@MCo!z8C1obUGWOlZHzpBZph%0IxpN5m%99E7XkP*tL4o`?jk2~l41TmJL z#&P&Nvpvtu&r)H03>ZQkR_g>snC#aW<5QQ)Ce}W5!rXC_KV!h1*%DPd5HIbL^@?JH zncU~C7cH8fu?oc4W9(93Y*z!E#S01HXxAl&ARuhHf>B%9tkzPlxgRMm#Tp1|r4gV~ zHUTaIo=telcp*88<;@q-z*u+l9xcxgG{9Ns;Lsdh|fLNa{;V4YU4x9cDOTQgXaa z3c(T4jU6WP(QA<2YXDqS0Vq@s5!Y)wyMV6KOsu+QT-ajDk~$2cX!8y0OqDrSIqQGN zj|XhAB|dM&NLpv6A+U6q41#pa_{xo#vR{5Z{l`B)em*|>_|H#A$G@DNpI%>IUtj*? z^6K=H|JN}_h?3voQaE@_paK5hS6L}b|BnQ}!)aIsj9-QeGv=6p#SxbuX3&7t|590t zb7NFiKj47-&Y6{JREUV(TIVYagu4@D#+@5N#oLugDI}?(d6G|^tZTPY+@YUXjubBv zBua4Ipgn)qdIFgg5fES&aCQc4wxPXX^qk09|Llal>p`|kpFn0W=f?X4yaDaTp)BVY z&+3dxJ6lj1hE-8a*}p1Ypv2qP`7AfOesx{FefwP&9>8Cshd;*AG=2@`WMLKQiM`F} zDfx?7+HgLS=!_0Zg)Olqrh-8+hrQgU2n!4n^v{N8qA!dD>1B5yaG2-nGz;GrY~(); z8;4b3zzgLid#r0lb$vrXM;pOSY1LI?#2XlHU=&Bri(ahg6tqs4Oim^um)wbB%5Ejc z+W>SLJiLcWGY$5{N_mMDib+(C<@!#&rs$WD?*=N8EIJKU9;U=fSk7iK;UFx*2pg}F z5HfG^uu?C&asyilaX_{C3v8iK&y%xcXTzidgF%4#n0wqn%pGcz$pPfm7BS+8r)vWl z!0Gf0tw!nPE$&N*0R*6$s5 zM_k+Xg8d_0&(Wj-y?{V*$ERc};!PO#bKTJl6TD*yn`HcsPpISX%u+B%N`Xl)f47{7UTvKw6Bp}oSfu2}Q-(l={IH4p1Rb}-mE zO%=gD#T@?jD>XhlF-m|w@nfelL-1~oo!%xmQV5gVq&^Qwx21A8Z42c`! z_?H(%Hd6N@-LAR_p&?~B`>oA|as=7YK8#!2&n`IG zIl@e$0~E{~z-Y)78Qqh20tyyI+;&VQhmg9(Z)i!2y`t|M1tImqoRP+lc{}P~iVv_ubpKUoHxI+nhuXRuE?V+ljiv*;w$xjafDp z*$v%|v1~HqBd^%FwCaU_CsvZGk!;bSQ~;#Ep%7T7nGpbq#GK*7q0Bbq1k0X8asNPp zhebM3fU zq>__r0vf7c1W8~fl@UpLIFJ=mPXc}tKu^GKMHp?i$}%Bx(SpMMuE)ZcqCM^1lgv~Y znR0dF`uU=N+#oLa;TJf2$NuP1jO6YK%71thg7Ql2&ti2h{>#->|7YQv@M``?_Vejg zu<9nb%e!9*8qp?;K}bX=qTthY?d9jyWFQRq8fWahHP;<;8*qq1IC#;7;Gr^pgj2oM zaKvxoIJ)?)r+6;Qjl34W=sTu|PAGO*Tijo#IN*`bHQb`_YMb?QVPL?4t=;aGK%G8* zC(w(-s%YK|cvmzpg?GJZ8hD2l&jZPWixuIfp3SbQ?BTk?rgB6C!+ zle4SiKHwAYo^!_WY{EOAn%9w4rB+0>B;dTb=N9`o$$$0GCbr7qeGWN2GISOEB(^hP>(8TxFOUH%XVXEpVv#&EP zIC_bNbD?kv_<&~IN2jAhT&QV3q#^CKHyFWGIwa$_%vVe=z2p?rmJfGkS?sLbnN5?# zr5W%g<2P&3n_0y|?T;5Qapa69L&Ahn<4_6-KF6RU4dwT~^eR6{kuJlsABnN~zYBRY zHf(iz={j;YQaBqK)kcTRo4~V`75n`6-eVz^aY!tY_W}#*lMxkkfYrmLB3EbVOsyE? ze6*)_8}=IS_B_pEb2r6_U{z}kxcDr51ko`F%Dm+M;8750?L)5@z$6)idD50W4tsK& znYfBhPLTJWFPMXj(XDqa14Psg5#G{uE=ia%hG1<yb}9KO$#B7s`Fp))m(a zc(~C~o-uvY^mDU+2(zrm+ln5lU#Xk9^F*#(@@CnAd}iXybu9mujiK?P0C4R*2taB#5@HSEx#b`}etY_ZW~vwS zulvj-`w7d3`>uyC30Q&ZED9~((A8UzS>_BZvQJJ}J|zBwZ9`+-!uRGA+vied3&YS^+6dM_fF*lf7oKp^_kQ zexV_16^l_)uu{?M3+h#~<86i0hax%~Vd9r6bb1kT=3-%+II%|sWLA`_lmT%W1*4Lv z_dtECMU&8^eQvwI_0eHv2EmYUlY`$&>_5Au2orFpTO!pSy5Yr{P?Dk4Ot=yQ-cK(i zuyuF{+12L`K(1zRxpuhs`<+_%aG){OA`oze#;|oivIUOV)_*B2O2r{$C z)1=c~X&vh$OGm&mDd%pI7af9`Gxxknj4L7wlNWHSjp)Qfyx~8d$XkRAt9KmX|65% zP|s*+2~bpJ^ouplZ#)e)lzCkytRK=(S5Lx-eg@yLG81G}#!%QCwSWN?mx(~!Y)5m+(G_ro=jm>=44%VBe&Busq!IyB_2>@e5%y7r3u zC{s+y7aQqQ3zvn?yPm|kf;V9%F25K1oG|87u0LV!gJAA@c>-s~a}^8c<_4}q~Z{UX>SHlpX)h_6EFkFcQ}JF)LUq{#Ofc(!qR{pK$0 z%Zcqg=wQ_}O`$NoPyg&N_4x9fe21TJsSKhtXn8(eE+uNcgVx;EZgH34TjD65?!g5eiV0QP}ip^(p~Ax~~EBOhL=>@yKPs@x;> zcUwt?k4v~*#(^bj>q)LZXIBm~@UAVjQ%SR3M$zr87Wtlq7m5r-%eCS$zT2 z$n_2)vP=WiB1;Ie|K5m!S?4Nc)-ZnFxAc+rxnj{Nj$0c^?xnho1q?eyhMCxMYK9$f zItP}*aUr+G4a=B$ULg&2Eb#V}vS-bpkd^nGzTSD~7jLBx`0C=rr`VE0Lk(ry=<(`D za7odH%9A0v8oEdp>zcWG2`1@HWY+^r=g@}=rL z6BABpt(zYN?6V!WON4Cs&adV1Qb5JbnYde2D^hLe?9HGvLPp%kQ0Uzi8olF63JBY{ zql#nEQhQ%GOQNe;DNK-24OIhXT!+s!Ii(>l1z<|h!nkTT zx67^Gi8#hZng=T@RcN3|hZ=svGj7JU;>tJ7*BLt( z+EDbkbD|I&+G=q}ge(Y^P<$pE+4Gk$We-d?CihBC@T8w+h>>?p-Enpg2`IQ2bnT#B zmveO=OyB!o!GE?v;JmNQl0GZrin!p8L9Mv``sAQZkb&>ncwq;xM$3p4+_EvUmH5$C zIGS5F!J@S)Lt{H$*gAl8DF_gOG$M?^eXvI&2ICGcJonu#+m24|iBvt2Dl87n4Fm*y zSyU^^lQDOa@s#Em3y(|Z0`kn&^Y|>ynV?)@SrgBYHc|vp?ULsd+Q*LHvT>-J7WGI= z0|QTMiB)=1SRppjm``x^b8~G}}a33OT#kSK0 zavjxG{TIp^7}?(s-n@GCltr-e`C76FuEHpuKQW3=jN-o(qqtSBLvN1lnCUUJl(D;P z+G^4Fj760AyDXz_q`l5oP9ubaJYM@xp|}~JSQ7Pla$I$b+Ba@ThB8aR-VIMlCBg#l zt(tKU^%SM&Fu9T=hNy^c{1GXF`@24(B_WrPGCY3VBUsfQnqUe-PrvX!=c3V*p|@xU zW~n9qTVljOmbz5@j%RGZ&U8XEI#r1^X)BH=bOgXvGArGfXehgHIf8*Dx+4joc8xXW z(YJ>^r&f3@^W_7hVoxzif#J!mzcwM^qu~MI8D_$%{QGEOT;v`P0?^T>W%>`u_Z)t~Nuu=OT*S zEMW`?s+79t@A7*tQhC26_eh%O$7&?ivVIe__MYq=yT>In3Dm76IusZaJT+Vgi*~h| z0g}qvPz}B0+5x=sO?+Gf?ovCH(42x!WS;P?Hj_}?0nouT-A|Rw5jeiGMieYt5j$e_ zEgYav6%2RNqD=v+dPm zKM{sQc2_H1-qDOAc#Y$O>07{FM?WGA`Z^u%Q*65OOi4njUy~R52bzU^MxXz zid^@@vG^1ng^(iG`}Ck-Q|OQsV1jcG%J_=W6f-`9#DW5fzV$+uy8wfh05a(K6+oPd z4~~b6&nZaJeR_LaxsKR`k5ZbqA&_SAnx;W~h%XhmjA@it8%{tOuTyc2{*v%t72+7R zE=%16-<_WVJt$o9B4N}xmt2HgCoSyK53c-D#X&7p$|3LR7>g8pze3T@CgUx?AbTCL z(vb>b0#B`Drh04Jxy=$lu=^KM*fX>6{+|c0$dE<1+-(rE9a~GG92A_G!e{ccd@*ju z_vmtpyf*lcc$YiO2TM&r79$CZ^7e8=Qpr&9S3;HD-+=rvU5?MxFKVTw6{n7z`eNad(87^EAc z$uJ(^UlxH@mWOJ|F_u-L&J*i04Wqoj{+ud=>c^<0A6F;OhxIcLb0 zQnxW6-k{HFe@wX1Eb9l4G54I;*;CAO-ZD(5jPi$!VlMhhv446`2{)ThaSv%try33G zld9)j!`^R20mDA$91Uj6mXvMu5Hgcv`ZSi7<(*|81v74<@Pw8#)Fa2b1fjhil%&a! zL0&gES|^Tu3{@MUJQWguXybhjrSSslgaJQAok4Rae5x_{Wy!b;ch39qKQxh9>T5#( z<-kgtiNil`x7&%r8NY)nXj2-$ctwV?P`X#GylbDKhR{1M8Im!o4LLj@BQ6RvIn1nq zGTW3YfS@d_sHJ9Dsj^ZJdPOyDMf3!dk&&Kq#{d>aKH6Z&jIsQ9!8@;P6Qx_EBGIUOZ6pC$J=~F5Ga8`-$cL-di>QUJ;w)p1z=lkY z+woWU^0Rm?UP0{3$O)A`<%IgkN@q%4i?#H|LwRE*S?}6io-E7aCyJ$LtJSD~g8IUx zs9P~Imd=^f7&gky9GLQmQbMgYb5MTcJ1r zj76vXBPc&NI_LFb^rw@{E1BFfv!Fy(YZpsZ!UXFDeF%9NLy|UcY|%8ZVJjcO%I8@x}2^S3jQobaDLtY-6lkH5Y$jc-G^> zXrp@P!hBFyTRxpKbBRsoKbQ`4z>ZS3hrnQCXfkwWWqbqYbxiEuxSHQkug?4VDV}Ex z?0zqkTQ=?hQS4xFGCXBb94551|3IlvD%a5e6B}$v5XLz!jMHZrx5p!|ALgLLTu7h9 zh4#o9#7vJE71~w#qght@Sobi_A(ByiaJG|CEK_wMuk5ZA8S9Wsh;A#NfCBSEo_%IX zVYb6+FOx#&dlvFo8Hc*mW{Q``pbMYoGa`Xu*=hiGs{xy|MceV z(G3?k7**u)OI^uo-z=xK(4rU zhpXZy5mHck>K1U}pB7~(QEULm22a6b6^uX0TxN1GzRDG&>4_BDC<~_f*_H_%M4FnR zya!ndz9_j3@EV{a>9~*qzhjmVVRu_G=#1#+t6O$IO|Fr;kplr zFgKL;m)@{!W)i{hdA;gU+}al-2DLU9wSDQ4OS?NIV~G?Fi_~b}g}t1>wv64R%rG_L zg4HZhu+bB1pR!^#*1H7?<>u* zIFO4yvllc%9pb~w<&`GFWX(Y*{ye;yOj>G$z zagoT8Lk3sw%Thn7<4mxw44Cci_~O*(C$HdWZNdX3iyZD!OHcz5m~#msNGYBju1i~) z)3KX4>x%SXEu(xww>Rm55g=kyeAP z$gcmR%rG<2TPvXZrcy`DduNiR(?9qdV(}9V?P?_=Q#McxQ=bE}1M<6^^o3w|4D5~E zF6X9zGYeAa|2A%yZ%*ZfG+x&HU(ZLHs3s|-2ZEZCDepUrR!HSjrO(wH*&(b;^c3i` zPkJ-RS+y!8f{};>7`l$ENxb`6+I^;$HU!wQi zRV>my^41=p&vcYq!5nN}FT1};!AO!7+A!52etR%i5V#$omm5qhRMr5hQl)2(iJ_Xj z*nhEqctp;HlV7QJAwx}Q9QMd>#LnQ*{r83Y?@RI<`HhfEa!KO<+W(N>NY?@(n(ASv zN=7O*fE_LXQOB;~;3jg!V5%?v(|-eIXhWoMUmotll=jFyu(Tt3ix>EDVhWns!FOMF z5BG=^)z9y^g#6=fhP#J*y~dPZv`o1&TbKnEW9WJ^zI#+Z7UO$54hZjWG0QuWAs-AO z8pnGlqq+?6mwSsAb~+~9^vJn+j$xC&w7aF3WvpZfc|FARSYA@|6@4FLa;H%^28)xd z(6mwuqIrtz2hr`?jMEB-DiDA`R&E``Q=a5<2_4IX5`Q-lcR7J9tI73F^_*SrfOH$O zb2;_3V;-DOrbo*tLhe|kW$KG1=g)pFXr?Xc8i1g!nk&JoEtsZ}T0HXV1Sz6(3GSfO zB^K+r#`|T z2NEExUZ)i)PTh&nRd5|RL@LLW2FLwdogM_WOywQ~9%vcj|Dscyy?g$8nLFXd(PG5j z{r6L4n_GAH^=>)KJ<#H{90JJdQfhm8;eyiK6waSmuzR4!(-swRzMfcj?0rHD&3t9+ zlxPb`&&R&peM!3HV9#QMK(qp2##a+mR&g%DL(2?V$Y`MF7-Cz!R30Uf6*6VVeULK8 zmonqY;(GUsMi&v$9m;!h@BB|-FTHC@)M(%SBWxBBia_h!T^neW!%w?cJrdP&J|bQ6 zS1F`ALN>y7sY@C+c{n`7Nmis~=s-kiwdDiOL8v*J)k_e4AfsLy2*i}m?J4(V1)KEA zxx}cZCfvS-<&qpAzMbsid;(0=FZiHHTh1gk}tW#IP*$&Z(XOw~UmLB?r!-hpJ55 z+Awb9J&QnXG7e?#I@f|ge)$2vv-EZ@DHtXFAsMnWcvk38|l@&Wro4g42Y-)|+3&1_;+mG;3qz>WL=dmrpdQ?e|2JT37KfG-<{vmJ8~;h$Hd* zdho%tMMcSmK)FRHg!FX?1=odpI_?Q(T=gm-kX-HudiI^7!UP7YMC1_J;7WW|YZid@ z4<`|xKi61_$cInlChj0%J#xi-JaA?3R3?R$fRKjTlb$j|d8T`0_c+UkbT2M1R`KFE zQBvywNQ@girG~nS(UV;`9xJ1bOPKqH`q78!N8Z(TLqZ8iEDQ+&72w1Mf)oa~p!R2!aro5|$%%1E_DBBGW~DzFyh_-|L_l zZUPF+!kP=^9N3NSC7So>z3wG)L{6^GNvilAQ!AG%q4%2MRBr54H0;i%qubj+SjnET z6#b{a6pC34SRA{2%Q{Ng=U6!k-eT&TD3d#2%?)rcK6!aN&V&ILM3m5O3U;Ji-y(Y0 zuj>f1`OWlmT++qHBOZ7wHl?rmW>NrWxETtdY{%d0Xtgk+_=%F z92)!rj*ZNj(1s?rJ*ft>E=Z@M*PaC*jrGr27klgG>Hf#FVQn;y|||;J-$3& zysm87HU^!442mCm<4e~EiRmFx@U)!r>ac@Yic`KgiS`YvC)Kb~E)`W{SE@~_@r>(n zMUerfk@A_oZ7W0l%&m6~Aos^|ahQh#T>-kg5?c;84dzKt0sKYuDddE5yOE3&SFw#( z*|`=8&Zr8*dZH`^YJVEON;0ZF2wK}ac21LYEa-?QA$zTo zcU*Jn^2I^FFgZ#tLncNo$0Hp}VIihZJZs3hk&%ddO%RzyjXO@m`p8mDr*K!1~ zpDdorGpUlixBNyiax$bsuxtZLwvB`H?a3P8n6b|@6vGqY8Jbq%I$(mSk`y%n8)y?t z_XChISq`;;WQog9l5f(A^W*AB4O3f`etF1>ADI(?FUs>xT&tbd4q(N zx7bq3+eQ!74gK!qY%6Ag?@rEY(^yQ~(Okw9#Ydik(!+5#y}>Q(ak);jz#w7e6`0y> z*)&Eq;I{_c;^rgE<>DBH?P_G28~Z?N4iXk-D=MA66+UT)Ts@$;0<;-I5iQJJrdY_y z2T4X)V_udjYZKRv2ATfi!zYk?K@41(a@zr+@W-Yx=C5wIR!0~pM7c1wVEM|bx$nct zxp$!1jp-_W3o(9%_i1AGNM2-F)0Sb%Vl*vWUaTRp>_m2l#qEOrrN|u{2-z!zi$A+8 zvdjhLgqSd28}G=N^4Um(Ki4(V;)Z4FT2G&HE!=9;;w%o%_aW3Iyaov zgdS+77){5%Cku42hp#f7K%*nDVLF?D00U+$3UmV&@P5#c!8@;l!)!NBXID$82R>Wu z9@KIMsgfPtDf%FYg5FV{L6C+NsD5L9IX!X=bAtlE(&MAulYk|-OKG8e42MTj9K!PO zF*QOxo&5xb%M`qnXqpnu2ZHx`LWLn2HB2Pq9(jv{mwAH}Gl~b{w4xIvH#E7GeT>2> zzju{BWT~Z8jB3ji*}<9Cmb-<67xXoFymiHk${%1~n$inz%E#D+Vj|57r+k!WEYc`h zhO8k0*r8nB39My?@t!^tC{Nw|8pTWMB*>-ouq%pa6<~~C1o(+r2xct0=60cAVYyj< z|MNe3cX8nkXm05-Bu^YMH7L{R3f^asplNKkIS!$`w;PPYNNuXRLh$^S`AL?g-FKDT zbFqe^%ODda;49R1A-h7_XL~(zRoq}bDd?g$qz**^N(jzeu`X_MTrXT1&cKcO)Kb3KV%Wh0L!>-?>@)15s?68~Pw$|0IVIzmqNU@?4wXKut)?LCEC#cQN ztguj;$SlXbihI~)+vwBPCan{GPz_4kBG$0GQki?HmNhz6o7WU8?%f*v`!1`)ET;Lc z&2gIA`*Yg!U)Nk@OJ9eRtMm8V6|wU3^3Dof2tNV;hAkjOWnC*FfiIuE`2L5&+mOZ- zb0cJ#rLj_!#zyT5=;@Ag{2S5SJ5oe$S(^u$pYG}~`?-)eW3%{)aqX%{<-gs-RTRR$ zkLh&H5C5Q~%uRLmO;9(BqpZ3~tHA$AQ>h6=ijY$_!H0kTefaq8>5M(FGk8$=fBg1z zb=d2KX6*dK;w3h*U3Cwx+{0#F+DW%YT#ittOjkF-_cwhb{8S!vGu%|yupvNVmn?Z& z-40*wbcSE-%EkmY<&A?FhFJrIar$y`(NTv2lB&HR^J>^0x|xCRe*O=P@vRlDm)em# zoe!B0?pwUrYTUZ!fO`e6*1)^|SQ%t7*0nmpX^|?L_uR+5yU{&;fDkXgPOeNrvjwD+ z);BALRpUKAcX0Jx)TmN-e?)UAatkT;6uoywjTFBOHHK&UHEgv!D$Wi}n)m(CDqFU;J1#nQ6m4~lg% z2AN}*TtYxPCRE8n__8FSYTf(Vz!xsJC>BotR4I)Ig=x4ThrL6e@BJf73Z?lS`^aee z0WwC01c~SUOP@3niwT`!5|iejA*T_HE)^+68M@+%`Nf5L?Pv2>uGu=d1gO=>Wm#4m zsl!%guiU0AUQeQ7y5@aiRKYs<=w5NdqOy6reVXa|&h1reUBV}GFZ)Kyj8U=dx@%au zFaVl(U6#bVA2K*pC2-v7EJ>1&aSId4#y;J3d{6)+owxv{Gp~;r6~V8>os9^xZGGYH z;IWSntqcadSMbwQb{v;g_bv7UmV%1fhE5T=w=fs;eONGHkooW;(0ifR{MH5;Jz7|kY? zt{G6wU8!mq;=QMuRI_yswXAclB2`(6!Y;MU!7lp&owh>8baAs5hQe_UaBG(QM8JXP zdJ1n5Q?l!w>m9Ni!c&!$3)E3t10c>m=Zi=1oPExzNLh+P%9^0^t*auV`G{v!Wsn^H z3TFwKg_*eHN@AwyJE|~;ah1%w`^jGS1C&0$YrYckyMn=#1k z9ec%uE3mBHF8koU$`nfXaLLu`#=^kgdv9Yb#ia=y3>4G$5II*?{!1n3`10(%ciGj% zGLMFdGM>f)99$r=f;Ay$S{jJ+wCzIdye^U<)T{#E?Z6<*;zf_LV8(ks1+P!5()m~9;T8x@xV$x#7aq7l{iAF zq8nxjx}6lpgF&5|>c&TWhi53!>yZn>4JIq8fZ2&v13ODK#>$NZHXmUWFUD;tMkzSS zE8MCMm?cZux+T3ULrtN3-`WvBvc6(^cx<3cZRjYk%+2BrDnUa0fdyZuIe3YrjviYE z6qtt&qE}8WE+O<=+TXG<@~qYEJF&Z0_EiSmG!F%qBrhjv0S*@{kQ}$yeXg{D$4nR% z%2dVZy=&Wje{%KJUtH6@P_UCIHliF<7QPGauG#udA=)`qscz7&8RtB)OnJvWa`7^X znL>pc^v%u$$r$Z-fi^(aoRle)bT2{Hh^2`JcyKLtM0@zgvVda;jDaP`iA))6H&m42 zhEO9%I7`$~jAa2NWY$J?nteMR;VmQPURF^7hQm6Z$ZaQ*i-f1l$34ejK#C}Zm)t|K zo|4wp$drXwr3_))8Sm6)VP=54ggX8GJF4BP+_^Wd|6Vy0SHYTnPApe}gFRvHgJABON&a-tx_Qa|$0`1eVd>lO zw!tws=T6htpK!Jd=4*pyU}FwaJ^EZ4iRPO)M7I7-!%hnRCC;}~42ahfoPF+*>mByl zyt>}$kn5e#eT{zzWB=-r^HHD)T=bbLYZLosmV(5V?X8&6kQKm)_Vd$a9iV-CFDrWv z!$rz}3yB8b1Jwch*Y4Hjx9|5Xiw6SU`6;1V%Y=ipAF7#N#hqW!EqRLQ`lzb`C9Ze1 zAXvnejg?+6<(~9RO-h`lD07sD;UAh~kGgHGokcshckb#z^mooL0u|F|2ImEf? z@x%ybv4y-NG|RkH4jxb}B@;&QMnuR6m4bPisV-KMm!!{~dTC_oPY${q<%-tt>b-5y zcU;vpf6BZDvyEH>^+A?l6_MY={Jzw6MhsJp+Ot-U+!onJyU{@gdD`@Rxh6y;7nIY0eKc2V#?{ErvC!#98F9rh0PUsw_R zdgsN#!QoMQ^OvKe{Wq_!clOFWWaS}r_I@3BbEw4P&Iw9smSR5F_Ny!J_XSpB_8AJ~ zqKUYip`DbjL|8j_X_xyiEj_5XCaTD`qnk;1sy4%pL(hv>jZ2j?#seajLAsu~^n3p! zq)GIZ*FZRG?lzH%G&X!2R~90{;?;o=cMt#>$9mCoV-6K-fDSpsIRKG~b8-5-Fa6RY zEeiv2oIpzL&H+z9FMO#|bUf>0NHa5^VtC$tBiqpBRta2%aGZf3=*MD;OKzi}czgRv z1kheA_J&hdgiRXsO&0KaN&4n{je@t}2E#%~;Ort@RGiN`aKjAkX6%k-7#jDN?Z_lB zqoq<@X*_}%*8r)MY2*EW@pWG+3&w#2vV;)nku#bMAu$s1bR2pB50B9e7;$`cluQnI zLNoH=W3L&+0~OKi=>%Cv3_9hd;5H04)6Heq?}#M}idppcj?DIhy_Nyu#KevVf`UXD zqjtrD!75yu|8i{S{_V#LL2ofu`02%+*=cyO-$G<=Ze-mgVrkXOl;eCUmDz;J(=1+S z!pMnu+m5n<3-pgE>^r!qYjq5Q&*E2X5XoOB}ZFQ<=LSYH>N5%suCaXMuG8irG^E< z+p4ickC6oV=})0P_{rlTJg^5(q$hqeBNOUcxI=tYM{|g^MjYp^-oi{&f8rmX_{Wyt zLtd?L`j)y=ZOiIqY9Cm_Z?T}fNaa4gh`)McPo2QkB*sc!_2sDhyBbe_8_N$sI)N zTiOb)o_=^sNy{nt764-`+TErQui>hxdn0D3MZ$M>p8^godGV)9BVm14>v-e100&qL)QzGY z4B!y1c|JCcb{Fkk zqnaUSEhT2f33>zUoESCWcTA96Nh1@Px&5go^q@o(Bc?z~lFEVC_d&6wQEQSXUrNzQ zO?80@sStW#6r&raFap(0wS!Vkd6duXrp@P z!hBFyD_A@IqLwKqfzYULwST_*>EyCQemK2^K=7-RPnUZ>0aHU|2HQ}}(iQD)9GUa! zsi|LX#^id(Ghf#`AYK;b_k{M5ys|J-hwvWj5q1Z; zc?iAHP<_e?XYdtJnY2gVO63jk9nfM}@{}vv)L@}UnH9oNH4eRN&?*A60cyr2!YC^I*exE1CVBI9D!Z{FYK#nxIwNYrT{_w zfXo7mqsR5m;laWG`7(D}FDroo8%};Mm>Of!DA>aLq0}q_6lAZ1_?f`UrGd(k8zgh& zn1~{ar8m3NXnqkAI))sPcE^QNs_y>!`(CX9{;QOtqfA&|gupUomkKdtG&95TrA*Hp za8Z<4c;9}Q*i}%HVeS!bnBo+U2h`vSb8v?>u%ry#9Ml z;to5O`-YxDhyd?fjahC&S`Lj$c?|m)=j@y7osW#Bq^qHBe5biRTM z-Dh|?@BbCY#2Xg4N379(Y{`^kUA&r*>nab!Jo7?s*pS}wD4%Mf%S5JS+(qzSt&i)t z3{)Aky^3o|t3n7TmTN1R08sKF)c^o|dA+mO0R@F+yvED~*rIMSkiA&fKPae5t=JQ# zNAVI99#Ry9a1(Gd-CVI;tfS^A2pQ@3D`lE1mby>(dgQM$;&_PZ0*fR&2wk3jTSXtt zI~ca61upiauLPTr%>}WyTr}8Is(@2$KvkMC4SH~&SGjUjAc4MJhm}^CaZCeFIA_YD z5_q6j<_yeVG;*AB2SR-8qybeSSs#|zbXCONI#vO22>&k6iA%kSA#UEJS(rkx9yx(r zWA>X{yMl1fr!3(bPN<^`Dc7jX2%pG)orHK7MkXM$2V{aLg-|S!1Hpe`X*h=>5+EL# zz%WlZ`$UOB3ihEt)iHPmabJUv?3^Q@jJ+r{|FJ4G%WL#4*jj!(oLm0#4IrwQVR
$12ixCBupCIN6Wp zwx@Vf&o#in;S|zZu6*ro=e>PyN4KAiXs#<8k4TE&bt1%p3(mHPG%|KQRE3}!&B_F1 z2t$`-EYfIlK{Ov|I9^=t2SiW1Bc=wdoAQ2tPA$XwPI8qYp7%H(*Nrrh-6s-5!YL2Q zQ}C!L2C3C#9|a4T18NW9-)En5$O`O*mg9?4h(PxHR2w^1M!G!4L5!L3sgU8zZRvHY z>sq|fOC2y3cbnQW;}lgM;!)D75~g38&&a%p9Od)cgqu`^waaa!!}yqv{d&vB9Uwiw zlbmt|IP9`-#_t>x7&f*;Ia#QiaXuG z8?)ofmjv8Ek(gz)o-i2OZ6#XomQABfJASU43FxzfFG>L%%L!t&fcez z=?iJ#$#(Mmz!VqH@q@IHW8QI~kh1TE1HbI^euKuI3(La0bvKBtw{r_Ay1)dr_Tj)K zRRvm8x^rEDZUokw6PA*+P)_18iW)ZH2?0)(&*9?wVF~W+HOx?BhV)F7Jqh0R+?Dk!E*E{5&_K1KA0^^r*7pxi0 z165$;95Yt1glSFH7!yR$|z%L+}U|SUQ<8<6mJjhiL3hBv#mK~An zpjO&W0DFIy)@wS1emo>Y35to3Hy>159z{Gj%Dk*Wf-Edp|N>XA(}bzg6s+v7;EFFWu&~ddoJ%L7by>L z-N2P(_h#!x#d7!$Kq}rdL+57g%O=h;jR~`Y4c!;TPot#kA%&WEYH&$zDk^VC%H_Vr zdth5Uy=7nN^mLAspmAKbbr*agO1lUsMi8?{G)L26>g(848GUCb2iAH_*k{9(06z$s zY4nDWin&5EMsaq>%1V4;7vQjswsFdX6+vQtPr_Q(*FR?94It zMfP!gSK$DM40A@&;N#nqmoH!b2S^UCyqkUe!>1E0VkPq;s}wYI`PV;|g}N^aFjp%u zCAFjOfE#bKOyp=(2yVvvP*{oI6h=~w=! zJ-)kQ3NofFV|JkL6iwJAQyg*-TFz!+th@lv4bTW;55XljFy}IO3JvUd{UXaS-Le&@ z8F3iJO_YYLc4zx-#`^QjAU69|R1buUg~Jq$ATr!Q#b#xgu&7?5$|mDz-n?3EXO<@u zr)^OA`1XXr!2%u$v}=|qP!kGW$C{?Vg4?c;;j2e2`}2ns#xY%fd&AD)35QB1<(akD zddDAbbO%%niYjYn3|vo;Qw1we2K#!Dj@aiM9l4P{xk-{;AxS2MW#aBQYE=M#LQ|jz z9hX$qa{n|L`q8Y|uoroJyYh*V{L>-Qm!#}-K{Nh~WlntN2B&sD)-VM^b?b-02nD*~ zP6bR*ve&{oRK9FQZx`k1$!MUiSoEGjp-xA%`-KA5HdGfzH@gm@nHhrg$4fTDpAX_x zTLEhB0}tg~ajAmb4yAnS;vD2swat8)PGkik+d4RpU{73-mj|9`;4W9{UE2KtJktzQ zj7Rw8iHC)q<^yDms<~cu8C#n6$g$g5YNA0=AgK)MDwPhYO4|n6y*mHy`}242_8P;X zArVGNGDby_cN`h>lPIa#@#Q(B?!_1=76J%dG$^JHFlf|{38;lB#woDRkRZn`%54PT zNXOL^qymy?3l5eJslfdUwXqnVhRKAfDDx#(_ub-sl~AW;=#@pn=3$o=b;FQqhnY;2 zL}1!AJ|7vWT=q|wdvh`+)mdCM8C%i(#T@NvbbJFgD>kr5Hg<_^LebQ!ZnIy~m0SuW z&xsu6Qh;}dJ(%v)hckmcg>fJyjJFXsX=4@n;ARnB!Lv@N!CJ|-V=><|2aet7k+&ti zc4W^T6z8<<`4Y&&Iz}G2N5}`@g}EjdA3hN#`cfqpwM&Bt7e>jnfDn2<5KzBT7^1ko zMN?{#+(ws0FTOa?mtjbiSB-RFZ0aIY%+LzhK3hIkXGPwem(U8`h5M>p7Z=;zUGBBX6QS|SA&5GfM#nkrfua??*ZSIi)4>_`mO!=M?|#NnSi zsH3iMryY|^2uA);7X~CSL$Spt@{XYMUoRgoUYd&-gD5%E4VuA-)6_6)d z*Vfx{o~l6dj4DK>?smq+zzj=UgdH`MO@u0H)Y+7Aa5p_46kk@2Z>B#A0rSD$@3tvL z$N-@W+v*H#w`{BqDfSETS41^v2qoouRI}xcT`y&-fRWTD8v~nTj}352 zLs@8c%UBNCAnj;qLNVjqx!|DAUpzOVNWIe;6e48`5>~~*aOAl3MsR5DH8{^PBU{H+ z*g;eb4;>d+#`gqzLi0QuW0gSgMY`TecsEJCdvz^{N_*q0Sd{GJn zf26p87B{>N90aipg9g{Gcg1vkfG=ASg~^livhFhO8CpjmGR@ftBQ=$lqqt)82uknR zJ@-07DE||bO{{ijQDKoz$y{;E00K@{9MgJ%UC0$5QN^K1t6?H_T5NB-A>{?nd{)%5 zt}-civPVvW51g}zubD0bFu#KT-v+A@_HHSZfR+^}M||Q?17%7^lOb+S6@j8KsW6LX za0vR8xd8g>o$^zR&5mC=&xeExDIjlDV0~OR=?g5?73|QKAjkaf4I?h+B*@3*thx~w zTy>s7u~=#rI`u*UYK|=rTUH@Hy}0`6-SM|)@3w?Tklk-9er5N9sgDe#pkxVG^mi^g z%mXoHIi6CuCxfOGB(>vr@4|Pn_5pG~fMDlTSyd&g-{109c>QtI^|@QPolZF!HkXrP`cVY3XS@LQNfLwOHvnSa$M9x&c2_ z&}b94pTBLE_U#r#<}}3U`mmx(`m?~c)jhtwy|gY&+>G6Q2M!>m-X3mHkLrNls8e!f z>U?u34U!u^;@a@Uq6T^-C*zK50DZblM_wYo2Qj4d-h{Mz!Ac-iOF(R}{9A%%?A_@+ zSbfBX_Q0`&p!S0l0{Av7#NU9cxs25Z=*KdOEV}3iidw!X*Yebv&Fez&BbwG6JvQhh z5luVV2)lxf>lLD{>beu%r6TP@b`iZbeS>CM{~Z^L=gB^`hz~lYBjP)8;l6A@lQB^q z_HleVsBgSUH=nR=+gNwO)5Q}ZmDTVaA8aoHu(?}d9oPjV4K+6j?-*0cY^6@DYIK)^ z=mJU)^oy+IQ9;}a#~?DZM@3BBahHNz{_rXG+L`x+Y=D<7c)C6wWaEC4)f;GTU&%_d z(jR~cw8slnZ8`lw#th_)b7r(L=)3Pl>6izsapg%MT8`rhPS=QWPV?*7CV?*FH>{RQ zy+bq*U*%M&Q3q3VtC0ck1mQz**>p=cnUV3+{V41ky=}W~1N){sZ!(8u__O&RLIW~+FGS@87w`|;X z)pEfp7Cj)GOOroqr7L)kTroBm*{a;jP9W$yfq+Y~|5dq$ZtzvG^n57qQ4vuUBD}7S zq5x=ymEaYTdV{0$9txcYb9c3Hai3<|iUP9qt@P#`bixsgr5a>k3$g2+*i;<(+^Y^6 zRv~r!s~0cG?hnElh{0NU26?9TBKJHrb8V+!bSc6tkA^i|0O>0PajYUNt+X;AUW=~j zdoI!jS=clKYL~0KcD`HFHZ7-noCh#pjM8mczH=Ku_@cPAPt&5LCPhb4a<6EfGesy8 zVQ_Z*gue7p|D(vU{s07qfq$OTLz;o*L6ZMDe*X@L`hQ)0xabk`H|!17iNRjqha662 z3{)Q+dmw^i7}AvnkJOwdw-ibr^qHESjKl$c!dImkHKm4iO{>G%A~2tnI{1%}pwOG9 z7ewNg4293y|Al}3{_OW>|F-p?b3E<(%p3d3kScrBWy-pJM(wrGO}S1iWf<=pHp(GE z&i+qBmRX$cnLL{@#=*hC!JAjF;J*h42etnm9=tes{cne_U%Yzx`o-5TzJC302QOZI z{pRa`BL~|6K%os)|8}tQSoOtyA-`XL-TyPW8%Kp~jA3 z7o~$ACs-lB57#CxKdM-l@evQv#9WhIr#(5nxU%2cTS4b6+3P(!?-Ru`Mzw=sPy+xI z@mUlN1Ik+{9%Co(W;-vWZIddrc|QQxzYGBO-%R{qt{up9+0&n3HAmmfzsb=&17NV1 z-baKQHV$*GJY9H#x#Gg~$kW71a@7>7nNV4bt>x$Ka>9pVc@#OdzPRSzV zTrfux+-qML7uryfu-rkH+L=3Ph`3d8BH;|o0`kFCNWcFyBcReT0_Z zx*4-sZVwmfbUm9X`e}ENEWU?>WZrFSd^zV%4U(nzra9zt+cBe`i95hYbigi)EW2Wf zVn!2`{Ylq?Sd&a9H1i%FQjtE*G96qw2ARAGujk&6`jF*T{HZaUqi@i?s%5hp-KEv^4^egjHotT(2kX*$5^`oDwMFJ8Z?>Hl7TeemW<|F?~wJJB&w za%?dZU;(Z@RW~&87amA7-q+!Xz$;pqp;Y`A^rv=4$h*U5w_K!0BVx7R0WBLXj3O z+sAd_(e(!ai~2*zDLi3fFozNXE6oPSX@x^${f&VVDOKt+cR1{84hBMYt!qT-RXg<0 zLDA**+;XJu@BkQM`G2P!oNjfIO5|C~IduY3?VP%&wR7s452?nJ(Gat|d%m_`ce~vO zKG{wXeXs-W4rOg`q79j`H+8|Mwn_EZwcUCI@UP32k8CFvq_Gp|we6|aL3 zU*diy^wd6;T&7FXP`@KRJCTW`$iDQCTnj#d)|C{sF&51^`)rti@;Vd8N(yb{$Ulri zIp$j6j1x3S>W5~KJi>D_NFLG2*d7X@Tj3h{k12gei1N_f@*nTF4*BnfLzcFQV&Dw< z@70@Ehc)@{>lZJdA6eh_12=dsalg)VMhu30Ctty#QzNc0*cYelFD2IIBk(1!WHrTKs4FS@RM zE;F8tCrAJ@`2WL~Zw~7G|Ld2ppZNc6{Mwzk+^$~Yt|Glk>-@g^$Vz^7YIG5|Zzn_^ zljc3XReb9r!mW1F)+;&K4lIE?a{hMDbj}jnFB>SPqY)tbb2`AGh3^zw8TmhSl(i2V zkqbh&Sa>|wWXS<@nR-`C95gEDeH3-LxVLFJ@|$vQ6&Tym?GiqXbFWxBgSEh)=eVvX z4btS_s`NikZ)TbQXVU+}gF5|x{qoJ>6aC-Dubuuo4t1iFZ!XhQu2lh%liwEUfEnMW zTe~Ije3nzNqW}U%h#G@I?Q&@tZ~eV^U7@T$hCWG{mF*QR#OJw>gQQ zE`#&pDv;1eRPwN#(`4p4qOJb^aJj!V1_{;dG5GETT;QPt#qPM>ufK9_q#D2DBW~JT zHowCj9_+p%K`L}ZM|fp_S!EWh&*LxUJSJ}~4qE@RWdj?%1qrEjsrJ{vjj*cnNG3y; z-0EV4|AaDlQK$Z`b4+R5XnX^X*f8hch#t}xuixlm)R=Lm*{t3@z%jOg{HgcJ?OE?z zs@WwqL$b49I=yvmQ@AE}_5fUy2d{!_(tgnT0b6=g{Q0uKRA>~2+aMTT#`14vZcrPI zoAF6Twf5nA*mL6*j?gMf+h5&-el9B5443GYJvQ}}YqB8PA6^wskNS>_TYZGtslwn% z0n3-du+QeGb;BBL=~cB=`-TpsHcr#jwrkPrs%C@tLHm2IF^N+nAka4a0kY>;hJr_v z2C9AE*wsnc%mp_mW#PIK3o6gsq1FzS#?VPiH(hs6SE$j{?V7%2dg6W^zKBntL+>N9 z6M+#P*_%vi4jFKM|NW@(+E7cvYN-S&(Z@UWQS5nWIn>x+9$r@GXW2nhjrvh70k~fEb)+r!1pmSD>{zQN)qjCso=q z0U_p|L!a`jQ0&uCF+G%7TAx7u{dz~wi6HM!e zz^lsG4~1CONo*ans>5#)wWn{uh+)|I%fVj`cIq=iCa8w#Ox#t+!mlnqoSyx3 zas2+Q(gTDwzg2QnKVgs_$~il)7GT_DtpmsT#EO*HlRPJSpTiR}bbl$ZKBP&Mw*M z`Y*p}qD1JV$}3Jlx-gXi&tuKY9yx5A3(7{_jM0>-E_1nq7a6+^13;5;p=W~v zzV3qlqAT<_w#$D?F33)&A@Wo@UJN`9O_&(ZJwyBPd=rejU1@%}C!Z*!0iL{P20mC} z9L4llB$q(z+s487UAzohkhRu+g_($cdYto1s%Vz6jO)?teX$gP=DeGijasJI?0C+3 zenq~MknW5)t;2WJ_=>qHW*GG9hygW>^fNkW^oyww{&dU69r7p40eW@nNuy>@IhqbBS_KxES^@2iemC?P$~JI=0`1; z`mzRy=8~b{QT-b4l3O;8C)aBMmFKhDzMANnBU2VKmxYaspRS+`ny6FlL+oCp3h__o zA8gK{jLdw=k#e-?;Zg%W8Kjr}Ep7kBL`&=O|9si-|9tuNlmF*de!n)7TRlSG&l1lb zwcqC;na}653FOWBdp5@U0)0Ip2;s@kb0fc1>Hj+XKM!8Ns{4Pwc=_sy{%_;gr2i9r zKL^R9^!o(r@Mvy>hx7RyB$NF;+qdq?*YoMOBK=>7|L1|F`8EH~S6@Hnf7r@zYJ_T? zrZ3WiHlmvjJ}9^yrlO8!TH+bnj~QznS8F)8c#6N~9KZ>6**sW*#5x#?aF}ZD#w~ zEH&s9_nH~bHA@XQ&8KFTM@`$kFwtFWL&&nIi`F`^Wg*9`wV})W?pW)=mj#@#mOz*} zMx5??HQVv3Wz>or)HElk2`*5R9H3^nKea8k)$wWWizaH*;_>ACod!us%UA7?sCom;=YHow(sT}(%NJ|{RjRpnwX-_2ow}w(5YQ}> z_m_&_@r(`FnNDa1d~P*j1U8E1^oD2L@VdWQs$_md{!Vs|-@V(Z6j}9d1s|m}Z+XV_ zSQ|E~I2piag*et1QrORGFI)T!TRjBH6!$}52V2}ee*gX1WDCt!ZT?p7*n+)(kUZJ? zAOCM9{a-!8Z^9Te>^}#u-_-SgZ(hE5vj1%5H_b`1ZuBpc&3VVGwJ83oepl<$0IsC} z3yZ3{ZXJ=|x`80gjI*uu_&Dj-O6T7vRdfj(ClP{K=l{*in*I0X!HcK*-`n~9I=v?0 z1Vg}vC_y~~=p&{H^{Gy#4t0<|E;6yw`N!f@boV*S~kgGzl$K?dju*L59qWeQBZ+DJJbPSsB{rNy&xSb<%!}?OOF1&+u z-&~3g{0mgHEM@6qC~D+4;>!l)GR2futit$c^0xm$cJLwd#UlvUnKFL;h||rqJXNI{E;eiatLLg(EP+Ea*8+qt+dXXtBtME243lC=OCx0E^Eon9Oxf(l z3>1U3P<~e~*j>QB2MONYZ20a1!>!M(_hi_6=-=}4zmg+nhOE#l$A7(kRoDN$e*N<0 zll;GpUtRt$ceT_3enl?BWAa`Kd+~WTnC!l1M)5@NRhZBbe#OP0mq^8=?j>?m2?lfh zJW&0|X&&OQruZxe=v2MLfBkjd5L(ZaRxc}T|5B||Cv#47$N-esu5AVgEr ziK00>FCB|qJLf<+!r}bGy6d1$hAeV*34D7?xNoR#Z)dxd4A2s3aP~P@OzZOg=#pO{ z#RkpuYSH|C&qQ>Hr&>qkTy)S8O&^iJ1H<-_L0SHu^DW=`$L}+`wbPv115Ch(QrCrd|Ov=>N;tFJ3nMKVCoSf41^#q5oAM z@P_durUw&lg0(t{%q~OoXo&x0m3a+pjky2i6Gr?xz1;i+hu%rACvI=u%DmPs-cE7; zjqmjgU~X^I%M32jlZV;bjg`sY_--~qw!ICU%7Q=4MsAcJ-1ug;gKje$`p{SULcgW? zzeenjN+($Yr}KX=-hBOaUHXBm+GoP0dhE&0MY`)rs1t4`DPvCu}2 zJ~~b1DFcrME=7etHBKbWd6(w7l6Nev8gS5drH}Ua2iy#c8!$OW$?2IGQjxF`6Grdn zMV9R!{^jcf=robSD4AtUbq7p9w>JgPQl?4m8aIMTgQ z`c>GK=aspdvBer~6c=&r55WB?J75u?a80OdimNCAZ68_ad_24Q6c7S7qYiz5d)ZKz zTVMgkMV~3$#J-Xv=#x21KKvueI1_d7>*8j_jotU33#N_Ts~$N48b@xJv+krlaxTaT z9kJ|$YPRtWu#nbW3)uO#pzh2Xe~eIzUik9`UUZFQ9<8M{CJgIeP^IWNdj5+PjT|8{ zO8h@5m?e7Nzto<7?8Kfh`_|1`_h!$n(Q||#ghmMENsygghp^tDN3=|Cnd#JeMEAfB zNjW1@kwhxit9{Z=k?=$#oT%#S`f0#lB6YpE5B@5f zb0W2wj0^Vdy3io>Ro=)zBc(QES1OWn^yt|mH_Y5KR=*0vn(rkbO6z1E1E%z)8c}m( z@7Xu6W{hL+;z7lsz5G2kgqj4msSwMlVxm!cbY`;&Zru2*pp|F%ENV?g! zQojz$`H`OsDhxMcWQI$ybnG3dn9e2Ii6?bzd5d_#gbQTzF?O#*c#rjnk%nf3LeVnM zc2DUtT;TMM@(j8)5?C|MQgSl}Pn!1KA?Rvp`RgO{|LtD?`)}R7|GT^U_k-?#{Nvwu zuY2&vpZEUn-f#XN|Gu}kyZiU=-+%Y%^6Vdc@3+5;VswlD`fc~`?Cc+NcCxqkfB!Ub zN}s!>2;AkubfxOzmNhb?#lNur$B9Q_cR1c8)eSDUF)5U#L0OZcwJg!|wi4j!@VWpCP70gO#PLoP#N@GJF4zW($ zt|s z(7eQA?!7a}lED&#+w z3c0#>r)QTR&rXg%ot=6nUa^t1JU18Q?(l{gdboFFr8(Oz$?OWFBV8v}eIg+3!}Hna z@=0sG0CjoNT0d#6pS0HQ_wbnVq_vKIPg?6It@V@E`pZ-oJ!!3nE-Clh%3y zoAso%-kR2WY1K45!NUh0&`H8_!_teo1Mm)J(DQ!0@Xy4abo7=cfB(lb?3JZIdY4=K zuo9}UHvfOu5hl~D1_I2A|9|!RuvY)$@b$q{{hzJ;X4L<1yNWpwT_iwS5BGE*aWQE4 z_+P~B+XIh0N-bR^#G^2x-s1{H@>yBylMYV*Y@nEq#+n*XSlbO~T1}-mXBIMh$uB>! zYrEvgP%L@dg~l83xAU-z`83XblAJSG75sUQEBs=ky5|3_i~cWP|Kae>*IysjhGjqJYxKn?RD0au~-u#tY zj&J;qk9f5%X?-$n4JIaM+&*<)fBy7oGpgiKpSh3K+@DjntI=(}=I*4`6Viecq?wvK z8M5S77bE=V2a#10g!{MqSea-lIy8$H=^=gb`i(9|jYVXd%^JpU#%`{-Fyv3YPi{YZ z-%`yksTq=;=_Ler_5hTVXECEjc71`3Sns8GZVh?$7f5(vf2q(dx#1h+#eX#=e=V1g z`97msyS%7wh{mgGjqnD<*Ps4>VmAMlO#bzy15Y_AtADhYL#d4uNv47_`mW05@;(UP z=bG%?NSWCYdAYGd%Y7bxH$>s6rx?LJZ}f_oXO#v)f&7j-2{5J zopK6|VhYXCD%BLqmKncep`d=OHyRp~f5&PTL6=l{$Y^GUzf@n zvW$*hm62LuislD%)%0I*9(~HQLa|Rn#q>~SwdxG@x0{$j&J>5U-Go8q)h5O^e8gm7 zS_iMbn`?B%Xv!Z9!K$M_AcoZ@@(?Iio5FT+tTyZN|5(2q=mQMZd;ec$Zc zHFfW(+Ryk6`|@XY8jz6 z>7;dlm%7oct@hBA_1%X;2sxcz6Uhz_Uw4CIV%h?09nZVK`UroXcY$`kQ`u_smWu&X zD9AO&&1I^){ql=%Ah2Rp=sKjsI>R6>&H%7ST@*pPh2r1zdB%?R_e&J7ezT@2UKwiG zDOX{Fl|H^Ar(A;u&3z8Egb|}?a?8bl+>FV9-`T&Ya2?o-f5PgZaX01ao0+r6y-{Ww zs@eAuW4-g2gTEZ?G!bq#nLAbk-2`4+&8>sxXSJ!@N!4gq5H&7=@mYLD71EBEM1_00 zlUXrh@2%`TS=X`xj{2Laih_unA(vxb_B`ZtDy=6bBB^&Y=o z@PmIe9iyXi#(T z$x4*@4{qC3=7YkeOue74zLLr9tm|~%1-ACvr0xxF#R;~x)3~3@v^#_DOwfd_PEzI} zD68N?7=Jb<%qA>|qwi@R&tzT3wmJ=K@g(mOsGrp~(O#`}38TqhKO=KFAZ)U*=4{)+ z8tQ8km=NtA*H1BBu}P+2t#-et(me_(>7~9&F}-cg-!Pg3+=Y*6y~pu2l54L9ex}B zIjsY4)+?uV$S>OAD{z4iQhtu^9<%NNsm^9Q6 z=cnr(so?31>wRthVJ(S=Cf~`kEx`?UGv{oE9ZYgD`-*%giDmwX;|9N@hCy;qnnI>i z|GYb}FZvm+Iy5LM28{gamW?~)Pnd7<=$kP4s83zB#_x3Fe)TuM{tBIt-+zZcv8R|3 zXtn9@bmhHZs;gKZy7;H$g6wn}>>qsfk^TSdz3p<_HkUZw&%6pe^0QMrYf^UJ+VM91 zkJolm&yzTw*llMg)2Sd5lCUNT769$2N&dI77qmC_8yo<9r$jok>}16cZ7hO#IXLIw zykjCtT{eZqGs{LOPJp|kf!AAA2U`Ay(k4qWDOPsLNs>D zgv&`tupkS&Cek0~P5Tbg8tF2~LS6cHnF{WPbiAv!VTo9m(B$HtYV^5)3>Wv*-b*9p zE$*y|3Rg*N%YjlNyuS3c935^+aLehcF~H{bAvVo}v=nJ91lJU7u$4x|pqhHCR+sNu zLujnYbq$d-W~e&}m{~O7EiAX6&|fK&q*G+hdY+#CUvEajh)!e1N%t|9#Q*F)d-AN} z|9$*?@A=06yN+iIo$IoDaBYL$S#&caF@g(_p~NHhuke%z&)GtkGb)fs6VNG;n2|6< z(~ym|uaCylUBpR|IB>R5Os2ZB>~_M*gnl4_rmcVMcnCaHEGD)RAW8^F zA&rUWc*hriUP#7?vxSaW6tNinc65OP%7x=isT{(8_4FNY{5K!Mf9(&m=}`U0{wc2G zVcyXg`&Vg#bhzguPuwKVBX5kaoJXFF)V~?0Q|HnD?`)ypFsCdP==AtRI9|dTlymcF zK=4pE=j^{7?^^gQAj7XA*rX@@zdJcReskhQL7y=e(*OScvnTu2```C}c(S?weJ#%x zYTjwb`N#V#@MF(=?Ct-tZEudC(bN+&O2Po5{>C_%lA&qrLR+^`LmG$wpOEvAyqJVo zy5g@CbFzc}@%~cRQB0DM`p|I60II!PXx?Rv+#7U&A}l1~O9sh5aCQ5oqc~?QAn2ur zA3C=1L)&LBk9YJnBLB)HRfL1UWigcO4TvN@TqZKZJh;J}I96Z1Ng^YSrdo*c z&@p^P0+hGq1mWPCa7hIT5Y~4MIXd{jLuW6KQNr0Z4TwOKkbI!ykd%9c;j)YR5nMnb zV2&8w(11AFI>#pHn#6(nISz^&2ew=~p}sEi3L~p^#e@XHbDT$yeqj+w@RS@pdW3#j zcuT)4=(LUd{OmYeTj)ZUoIO&DbiTa1(^yCxhW1hyV1Mex7J7ShihdzsgmOQm<6O>& z1u0zal7SXlYUYWk0x5j&vnUBEj%A?}&yMM+j^OlDBU@WrX=@y3G#ZVum^n)DLR5dK zB!A@Q*`6&lj9k~}1WQtA$$Y9{<#UDeS3e3@NIa$S)Qu;>V&0Ej*IXlo_PjlB&k%^3 zpyON+0*iQGn*AT!Tcy8ukWeVE&Ri0*8$dC&B%b?pd=xiM@l?T zFvk&*ge#O%`}3Miu|KbG(MzVkl{6w~OhEXMm2DG}hGF~Wp%D6Zwtu<2Jm0Afv5O>Q zp5y$C9z8mHdwHU;`qJE{WgXTxs{b)8fD<1}eFF^`!czrT51mQ`pZF|_NF0#B`56F` zLcD-vNZe5A&(y!*x|$hj!blB^Ep(x)$(tQ=oJ*D=OMVv@5~WrL73@;Ig1ug$mnEwT zEqR8_C~FfDG#Cp#>X5QJ0xOJbMYUVt_B~>vhA!Nf#1d$;WFs1yqB%=}WkQ+mP|3j- zYhO|66mUziU)!g>If@f6-_0qNriWWI$D!i~EubIfpctx6t?#VY*6EjQ?3NPPORVv~ zSc-fcgWrkMfS3#8G)zY3_P9BY!V!SXP&>jP>TAa9r$VyGd|0IiyCiplep1TJP_m)^ zU|8t<=klU}GuPOz`uUDn?63zj)6sRl=W=tqq)%pTX< zpp)plGVn&uHr!}7+8?PAB;)2nr-FQLg)lY&Lj zUJE8+2zba+dEi*o8To{U+xuvU%Fx_F|NB1*6I_K?NN4ZjU=Cf6JoLL2YX#vHhfV{G z;mQL&oR-z5oZSd8_+TE+H37g5K>VoIQm~-m+#w%)rCT73p1Bc?Q<*QRMs$WiU#dgl zde8d-`e9)}xZ|7yjF2G20hvJ|EHFE**Gy9ykX&QzEefgOhEttT%q0m5C#Q?!z)c07 z5^pA>aH~A>j^`Ze8)B8N7%B+`U@37NU>>0NrD+xgmp`_*Xr|G)azcJZRcj$j)7a`O7k zpQmRRmxr%kLuAj3coH~U=-0R?9kDQg!Sm_pBPfFo`e&`9(&6$?JsBs{>-60tL@Rk(yh56<<&^HXRSGl^`9bgpSoTo~G)#-){}>Ab85=$i1pEUo zhvk75jXRRzgG8AP8Lo{CpZ>e!913k;)V;&6u&&%PV=qlhIE^r$7muun$QcXlF6}zz zc6FfKGA&cFA{aZa z&}h(taxEcmBlHO{8~Q9-X!C&zsXwKG4#OKf7tq5ge61Rqf&+!mbskdwT};FBpb}3# zbgD}&8dnJkP%U4{nv7FH0x$2skei1-p-?H_pU~+2AC+kd-NZxhv%B7#G#3TAfcawR zv)CtzgnyVMupx{mwv0JR0TzBy{hRJx7Q5I@Sg@mKShNO}fd^pAUIAhjuDa7`uX?c2 zQeAq%neM2*nxnr`9BN?VB+>obNdf)#nVBYdS+>@igC~~EL;snb7rT{@r~qdkk7%sV z6|f8G)fFSxKbiaA)GR)JR|{so$*;s2aW2q&>mhNZCh9knif?hOM|kTUbx zWl?f4;f9>yPiV|os3r|E5kODTpT35vaNh3+=0F3Z&3fdji}1<1LS zhrldi&gxybW>gah7?F&bNe8NwhsVB5aR}4H@s#WW)v*|*F$AWNAg?^5TbPB{qq*ux zb6LUS^WrzfD|hX}z~#gOI|LI_Ta1nr$4-+1Xq9NBhBztjQYHdbIlt;rGvSbp?}7(U zq2F41aGeEB(5Tu7s~EDE<=gZ zNX6vu2G;g2jm<6p=40Ej`tEySt6t^ozt*`<_|CgF@<6>~5CYW~;PXXP@d@xJBRVOj zu!H8qQ?i>Un-{P`y(!HhuQ^cSU`N^>`jT=Xf2VTxiyj~okPOh2dip-=q8sg*M1SGt zCBcS4EEIvc=G|}${&4lrs_UK7q_a{JjdGNkTh*s~2k2YqzktA{shZp<8UvF4xcB3p zn&!FsRY`F(s-5<5!7FHZa2!NLa_VcT+vNBn5(hfirqqAtQELFksePx=&qKco?Jl;S zzQPHbBd8ALiPBKgBqZP;3_}tY6r%F@oQQespF@&tzRtpK9G;&77OP{MaHQLttXdCU z(uknIh8CF>XwS>wSv1&1gZ-KM@8j9iXVIVx+JMA!D7SJ}=L7J2aaEEb)!x%@M5a>- zf=q0w5gC*OFjivnIHY2xA(m-MbuQF4fdV568|GrVUpO6z)e1v)LjpvuNvsH+DTaST zh2rlq)dJ+G4v$fJqHvU4YDKPriy6xhcuzpxqw%!VK|Bc#6MAlNfr_ZM(@NNh@CO>D z5sK4jOgMN9S!^zW*PR=eU1b+7Vin6FHAugO*WDP6qTXvtZia@-xC+Rw?pF*8N-i8Z zkb26hr^0bIy9(Cl9_t(y`+)P=>UAK9pd2)2_&{XB-tsXT%a_!ka?S!&X|@nL5{oM_ zw5*E&*Ub)@p{>FJKame|xFv`L$58c+J7L^NMRu))CFNpl#->X3NvhQH{xqhtP+_Fo z2CIDy84koyjW<+dxCO~>!M4`sS1&(Ofb;;U@SM|Y8j>kF5kA(=w7~(;(C)y;2_Dms zN=n4w;Nu|REEycU9~{1ZJ^17E?!e~Hy<<$iq#@Danu7y4s0Tca55*aa)m#0W-rIZJ z*X+}#-E9_?$_0EpZ4c@szhYZD2x0wkL0~avr9~s*hr&0it7xBcTvZthY_zB|R(sST z1-X5QXh*k!btd^?-3b+F3RX9z$fm*?L#+`d*KJFjt)VbnmD}fA2$+#^JQfD%o#u2G zMCQ!yxD*q^iNYQay>A+;FXiW=g(w=dw`||;On2^vs`s7JQpebDN7Ezy;{Ch-JUm)B zsa!6^P&~uMshq{|-@k`yjNwitf?9#4MYG8)s~*g^g3yDAM2yb|2ZN`t=m5~$R#ivu z^#`5x-+Vs92*-Fz@`CC`Cc4GRUnvn%Kn#K&W2fm9Yi0`hdTc+H75PazI2i2j?L~v# ze0lqie|%G0pzYy-EwPj;*}#7wvJDOfgU|Yqzs$$KJsdzlg`LWmGwinhgayY`@Dxlk|Ct6;BI{}exN@vF*AZ{n znF}o)U{v~|Wb|r5hXupwqShs{fV{=Qf%(_q;K#k)LBg42J_`p2gUh4y!RKB3^6Aqj z3*J8K@$%Q>;>+ia9sglSO4qNxz2T))A)OL2+_Jw&jwci9yAv8pqL5$#je+)O@YVoQ z=X@&83i^*^Xp+V{h#qrtfR4{D(Dt(@!!$^C(X%H*=_k8jD*#IfN83N{>EE6{H7}o= zf9+Vi*V}&`gY@$@A+eaz30TImX6;*T3z#q5>w$vkpdIk0TKIiKzOcv#BL**;qzMXc zn4dO;tYT_`QmC&6*5_T?fwo=zcG*!+C*f%vIb58vBw zlzgaqfDGr84~fDZ1v_YR>x1?*wx_J>XfXKWvr;y12%XO4;9&prAD_+Mn>DOZ`mhj# zlNiU6y7rfD*a|MWG=ZXYMX0-qzw37>ftQt>#@y9o5-q<#EzWJAw-3#%ve*Di<7C;8|*U= zW2TViQSXv`u3>nD+b-u68ybtbK@DxyWpkCwui3`79Pd zm)FFWlFSuUy(|M%)shCldG$+X5@n^1;wv+>Vo}g4BEL2-A1o$kjhj$;d1={o;_|}k ztE)%mMfGJ@h0RNf=T?WYnn4khO~KBds9Z241A1os0ZJIE-L!H}oqVnQG zu&c-Ag~e`Hh0M$9%-$b1FQBabJ3lD0a1k91zWQ*H@+B-?!$nLMUQ4)0t%7MmRT?BL z80-!LF#tvAtrogk=ko=3ZOJ zs={Op##C&e4R<80E=;C&FTy>9$<(gq=oKcDC%fKVm`p85rg|SsV~~uQ+1&-n6e=0E z2g#HR2(La!X3>pH_iiJprgc|9@)dgjb967m|DkyQ+vd0N{&(^IFFXOhMLoFl6)Own z$rb4iT`nP6q3Y1#nw2ls9J<{hvlhjn@v@9>Mh)(M-Ae}SXLpUrYg2-|-1xF8RDKXj zaL3Da)~Ey*4%$195o;dl!8pg7V#J!K+!iBtc0ci9i!KiB6CY;cvGUlktg2BK8U`SP zIwqWj;TZc@WyHSkxUB|NyI2;#K>>zKPPAGIg^q_xIUNe_BMQ`0=B6GxW<-+^sH!YB z^*wc*I7=n6HS^Ko*)hUEX)YwYsTUPC&@H0u+5Y2t$klztlr0oBJI0hfD@Bw&fBKLD z$`-Eg;RTen=}j#GW$TUhYMb1HVfl+DdC&0>t*~p1hq(7(g0|@{PRDJDtzMdsyJ*D9 zZP;c0Q>yBwqdtno&>-IR-v&B1LpM^h)jc%~UN8_DZvjjfBQuUkt65wFP_VpQoUQt& ztJjWM3Pm0AV_FygV{uj$P5p@Dxh@H}-ChN+Hu(O?)+(&k#{B zD-@-(#o7c4=@o&Z|COR`JFBQhh)UH6F`DSW6*Yb65>JSSl$g@ANDy71NMK8iKo4E& z%Dla+0dBX{0_>Xeto2NDTxmn-P$P`_%N4}L#oP1ej^Z#2w6tKjO&n&gXuuMKda!Yr zja&Vd$6;2EhRo4>$r2W_>D<+Hdpw{S9%wX=LpA; z9Bga8>$8(_sM^G193shBxC6F8lKJ%?|tOD%M~n zT{7f@(VZ*H=`{JphEu*q!)fz$@6&`@o6CLMQoHfdI|~r04y4u=5~<~;b1Y{~i1bTl z$F)0PKC{)uCNyl>d(-fo1@;6L7tFuvKaybnl}STPSdbTFwCiggUIY0wmV{s9u*@W% z@9FA=P?ALfGbAKIVF%Zrkbg3W!MRZv{bx`wd?4+IVF z?l8CoXK)Md5NvRU;O=gLBm~zC$>0)PCO~kPA$WkmU_l0#!`}ZsyXxGWuD81S`+Bw9 ztcz7o_q!xK;8HPE-1l%3!xuGx7!`E&N|Oj_rx)Hoc`2nS_mV35Y{j!a>QVQtu_MJ7_^k0$-PWg=y@FkYSHD1%BaRrWE_AVv`j^7r*_yR}I zZ7OPK$#j4Ykc}Z&GS^-mK~e6=2S5Q)foF)j@Q1exUSyCuB2P3CZi_B#O`NrvSNWao zqi2j%pWrmDTS73uo%dFPrvs8}BP!;R$LJRme5=TsWnC+HbmZ5ExhWd+nDxuC{(>85 zsM8!A{Ul54KDA&9VpVWk^~A`#IZrk3B~QI_Q;)}>;z;s0A9?pxaJe+J?@izILBh=V z_GYabtiXsG7^x}UcCTtf%Id^TFr8L~YHHWzp%J$y!_w~2QsMS0@BY^G00p9yW5-VF z4Kpnzer9Fe&%is*d!gK5B~)$#7`A@?Ld!127vBBVJk+Ly<$Q)lk=aDe!lsAg!C?YN zq^R>^4fLSP^%|M5%rUj(N6hwauz-F-!tBm7i<6#3fyl?Mstd31km)+`h2uIH^s ziT&Yb^j6lZ3zSvNM0UngfE1R+aaU_Ua<-0816=0+ezG+(3>KQ_7tT28;16hu@nngp zcor3ZyAt{f3ML}o3^-V25xl0ZKdJD-@nKD1NxIquC4)(tl8p^;$=j6_(@<+R{@A2~DqUv!i=;q+s?{f8m1{)z?NPgd>S*_`+s(SFq+7LuG`h1~9rVQ@4`t8SS119yXO5K{ z_pdPZ8F>M}oIJ?3G^#tNl9iZy^-psL$~xba8L&gQd1>Y=*118BFXTO|a;S_<8`y^} zS1#S54|~AJFlEI}6!ilfPM`{FZSq6mp>*}+%7lg)W$NQz7ttrNog<&#OlLE}62lg4t{*R8TO*hV*+ms;5dj@?z4 z%kAy{M0d{K2m77yt=rhuqae)ggI1pVn)AD`I|bxjSl1&g_F>_KyE1HNCThbI;ZqK@ zxW}@rm%=8qQ`a7-{c{))G@uBkIz7I7n>|e>({b!(@@wY91Js&1kUzZty!#Ox zc5H&YTMMcee_S{XzWX?(GRl17$!S7HUN+HN_^{D7eqsi`xFE`>F0`b5tp{cZbdpj* zQ2UW8NmSqZCG1E;1Q`jAw#V7|CN6Kqf{&>MoUVv0-fAJ2eLBIylOGDup?GF<&@k%s+-in;;AEr zoxbld74zkjPWnDV2O>X`X*$&d%8rdNU%=r#9E-4tlo6y%p{{J5p7{?5sg0$5!XW&Z z5aE;A1>P)*cDYP(vBu_;<%US2Q7=X2jlLPG^M3=7<0*BsG_^tNw{PPUZ&{h~VHYb3 z4CjZgkKO?RUHUY(vV%I$QLZ zE*5Yz9AkU}C7onbltp3l$>ix$BPP}zUCqVVwHOzBK&-(l)n`c1Ro4owJ(?3W<_UeA zqXkf6qOLDM6@rETg**=8B%Xh5XeagYh*-XiPoQi=ZDnOp2md_TOmT8b+~HOdm~!xv zwHFwwy&%ojLT&M+whz$~i!vtE>IfFCG{~hr;LWu-Gcj11Y<2_R6+JtLQIjY%L6G^# zC5ijYl$9mUg?yj9M_Z0lFFbNDFG|9oeexdrvd6zWVmQcWma$7O=N&OUEoDhFm3Cm9 z(u*vu#OL~v+pg?zoiKiG@Pki)Zci>K*yujksN*t|0ad0&oN=*md)I_}S46b7{&l(# z5r-ld&#dBROJ6UkqWsp88TtwN6Oh|*J0L!jXy@W10EC@fb;&h0#e~KPHf8%f2y$8B zcBkg*dK)l@4w=UC0MGfSK#36+RJDSt?opo@`CSV_U#;<*J6IjY zl*c6Xsb@PZ5%V!y?~yZ)seuDNE&kG3k(vyYXJi+&%YSujpsL6c9y)7m0O!hvA%}aY z#C5I+sqZRItDQJHad{6-uQwj~2b*Tv>G?$TR6Ch#e&5M33E+rz&cFSn*|~jCB~P!{ z-dY_L1)*jbn5#PxdC3Y;BPSsm7a@_SCFNvD`!9$13KB5e18(F>B(GN}P<ys!E?!9tbU*W^xY`2OpaZkSDnnrL9U&s64S8KL8p1 zV8A4yP{cRa>ecz@9_7B4pMKeEylG04a5LPvn2>C-SPSoC1>h=EqQd3-#(UKT3@F(P zBdm|W|AQa~A|iE?S@AYmP;!U{QFl-St#Tc7o3HJHk{nD4qHoYemm#k=RJN)f5052N zl3;T2>u}uRwFG4A&(mRgmePJCBKW)tBHmravoO7ncWXr3my+r((T`+g70r8 z8epPqe4D^^*J7*RB`E!UD9!&61#;>scM50vPkM9=;h)#G$9lS&fRJd)_P1kU;4h*tFVm*W5?0X z$+AziD6I^Az|CwJ`D};jMa{D|2@CNon>8nNqXZ*q?jp4UhY|;^=eKG2G*Vu9GmCd< zeexSO@RuZ2&`ET%_+VA6N}SK0yS+o9{-HRBwyYWW)|3+sfk$7_HX{cM)H7y+Rjuiqfxe` zgScTw?sZF!GhjjA>0FVL<(T=?PWhv~uT*4v5!x+c4)v46RJO!vM$z381TSGX7>PFd`v{8$^sn}bYI%JYL3v^;%e#XoeVtJ2?;VSg4a3sjtQ&t*4t|nV)OtJAMI@w;J z3H?E>6^n|Hr-`D)pkp5^5L99QVbrNDuAYT%5CXmB|GnQzZjW^(TqghU^=3zylxUUJ zX4uQP0HcEtP*B0JQ*QJ^zRwX`{3n+M=8O6gLJgSk*S6I`8$sA@n^2fC<2T0yLUq+( z|InLXSxM6qSfuhC1e0u{w=|@c1p1FA2 zmQUx|G{0kzuK_B;97AxaWZm=RbbyQl#qSgj`Y0@p#QS`HVEgmVe4i}zQZO)egsfpI zJ*KfRvjEEkvdd^pYZ=pBG>&3=g2 zcYU}$x3s@XzsLa>76;Dv1OLEzjf)RfhWg*8y4C8l+RK7WqmowWo@r8Q_f!CtcH~FA zRhqt&=#>-2gYh_ECIe9WIrp@1c#m|L^%Q;;zLp(0nu&;(Lk`p#RQ^usy9=?zpMi6-#s%{2sx4evdf_(c zkU=g{Ll8w5sd@Tr{I4$s@vwFV*?!V~z2bek4rY5x%WjMDeIvaOl0$9>@M2dOUypj{ zHNTjPg=XZeSa8TJQUu(faB)cvh4i?ip16`Z37}UL>ly1zqC1( z_;a*>n~QlU#ilbZB`zh4joh@gmzc*&E7}6;HFx`6+ckgF#C`GQCVIn7-joiERkhS9 z70aBmBwzN%lzg0-s*;$W+#0xX6MxO97-z}QB6utEv&`L^2E(|dP7}i^Pc5WgFQQtt zWKdd4_E5+k6H(Q>V^-!IgT&wZhV@Nj$f2AkYBe|04Ln!{xt=|5Kd?qSmigRx^hxW` zpl|?gKIR--0(fsA5PKKg2|`#3$mjt}WR{E3<|4WA5Qfi-o}tVUh{!S0)qgr>j8E=i zr(c-$$VNM#d6wr;#8RRSar2mm6sDd=gf3|rG9;hX0OjF0C%89q&=nW4Zr#lE=5 ziVwP}?Bt3U^lv<`c8utLPT|n2?zUI0P|6j^t2ju+t})%JtAPjNjx~u;i;S3*la5)R zievi=P|y8M9A{B_T&PqC`Y`Qt&AtIZ~+cjTYY++h_mhM0h?kbLA#Q*Mh@UcYp+*5$1g?^{{uZCL}p*JT&aGWMA7T z7e`Q@m#Hc_zi=%Ph8oHd;pFwQxe=WZSyXu0vN0s!Wa$gN+*=gZ*Kerj`L+Gmy$uU| zT|Z~r;a~9yyA?}?srpkZuP!kOuZ#GCd>gtVaz;+&0Hh#4&;FpS{gC;W9R|0ZB8sk5 zd;OtSNGYF;CU#?5PEFd)Y*B8V887%$&daNKgo=|x^!F?HcMQHB;O#3UvwF{dMNp|_ zpzA<>8+)1N118t@U<~3WEAp#9oatP8C7hp3Y9xE6VA#t5p2M5e37uc1b4KdRSOF zAkS~4@34Gl#Xb13ehmJyIowqAia+w7BDhAvGOKyGKHrMe?G`HF+hd?NxDy)^B+Pud zWi|&dZ4H4y!2|X+7nLWHf-H2Iw1s)WMJ03T)91YcOqd^f?CgDt;&PrO|=!+r^>E z1<=^@fz@ozY;P@Fit!B9tZ2EA(A@%@^G@B4?zlS3p%o+h%ckWJ&3ALD$5kO6V?8b5 zqTE3=D3AE-Ape5w?^GrM)BvwjS_P8=kAcw|uIXwy363)hPCSAop1PbBS|%4Y`D8O9 zNV1z2P*KL($|jnKR*1qagZ%3rht^%TK~$GWKzM~;089|wcgBS@71_Hp^Ge`5&hTJl z{7$w_f_!9(x$E;|8D+MFuB&CVnf6x?Bif>u(IMtP+Gv9ESD0>E@rT}ICnU;AGb}gl zs^CDqZ=*M@X(&Q~54^Q{K~{zpTgLAXcuW1FUyiQhsoV=^$gD@J_kfE(muR>tT0qaA zvxph@S7Z&xcB#54w4__Mk5v0Hwq9hI1*t;s?dMmCK-vLF!)X@+lcu<`B?YZRhVqL{ z-5}r>>Ft@WY3F0&vNZcZGl4zc^68)qV30!V2dO|a{u)g(ix%cKWpVJB-mb++ku=NX zFPG;0@(G!GN0p(5IE^`9a7$@#^Vy_Dcw?})JG^yjsT$UpqxmI^eB1DW_>jRlKS+B5 zNM|)GuN62Ln%RDtzO4N7h}G2ZH17O7`W4?G(G6k)eeY-M&9$`q?~~mi|C*_hI<3b6 ziy)C3r>4;y`@NPk&VZU?^ znK{(v;?trlaHN)G*8Y3}w3ut!wNKawEI3GDX+6H>N;#*R_ zWbEksg%0v<5YcTL6sBSfZai5-v-WdY=!PnFx6jTiIhfV`nAf!2XNLM_&jqTw8tFsUVnZSGN9pHsMo z5G|T+)ieRE{Mdqgb2@H7Ry1YR)z~HK%+*?g%H$2Ff}l$ryx|ixipo42Lw>STfa}ZW-XOXxY6+=+Zl936X$|J~xd zU4k-?52ME;SZPPLJM4t`?)r>Q67EK;<1NA}S}XzcNcfZV)66kMq+>Uo-=cDBCu2EK z@WYG^J42m@QlWVt?o~)|f9nGH)NX*)GpsVBgbsz3*oVxzVoN@kR|y>k>Cuw2Wh1v{zZ4F1G?(Shwg7gsx;p| znS}rxIkmcxmd@8H>nk95$(uRxAABJ#?Xw%mj&6r%q56BJ53Zac|EN*svl`u##wIGJ zRhmwos`15=PGhBrEk z4DIl0zT&;ZOeaH!h%Ug;nFQoz1;&6Fhd{hbW--Nlu;Xi1;>{6M+ayIi>IvOGw6Py( z${2~C6f2r7TI^?*5Qr^Tj(eH@Wdtu%axDAq|RdxQ`+K9br=Br3NArD z_1+P;BY&G`ah7Kv0E}^RwaF7mSc>{DX9iv4V~&(~3H-)=c9m=@ZbCyk?qA3HMWSSJ zK42K+y<+7L#~>H6;Z1keWEErDA{6z_Dh?=GI*omypzH`)O^ke@mzHARCJNp)cicEp zx_QR4YvX+K$!)A@`{trvQfFCxBTt!-DBohhZ;rx3%Q@im%KWXZ_*lHLW@uo6wwl|sp||_A&o>*WFk~x23yxe uW9bg+#alkx{&v!8(*TyjW08p+xYXxApZy=KP*6}FA3L)xI#9p_DE|X7Gl9+k literal 0 HcmV?d00001 diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/.helmignore b/packs/nvidia-network-operator-26.7.0/charts/network-operator/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/Chart.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/Chart.yaml new file mode 100644 index 00000000..a5493b90 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: network-operator +version: 26.7.0 +kubeVersion: '>= 1.32.0-0' +appVersion: v26.7.0 +description: Nvidia network operator +type: application +keywords: + - gpu-direct + - rdma +home: https://mellanox.github.io/network-operator +sources: + - https://github.com/Mellanox/network-operator +dependencies: + - condition: nfd.enabled + name: node-feature-discovery + repository: '' + version: 0.2.1 + - condition: sriovNetworkOperator.enabled + name: sriov-network-operator + repository: '' + version: 0.1.0 + - condition: maintenanceOperator.enabled + name: maintenance-operator-chart + repository: '' + version: 0.0.1 diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/README.md b/packs/nvidia-network-operator-26.7.0/charts/network-operator/README.md new file mode 100644 index 00000000..f25afb80 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/README.md @@ -0,0 +1,274 @@ +# Nvidia Network Operator Helm Chart + +Nvidia Network Operator Helm Chart provides an easy way to install and manage the lifecycle of Nvidia network operator. + +## Nvidia Network Operator + +Nvidia Network Operator +leverages [Kubernetes CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) +and [Operator SDK](https://github.com/operator-framework/operator-sdk) to manage Networking related Components in order +to enable Fast networking, RDMA and GPUDirect for workloads in a Kubernetes cluster. Network Operator works in +conjunction with [GPU-Operator](https://github.com/NVIDIA/gpu-operator) to enable GPU-Direct RDMA on compatible systems. + +The Goal of Network Operator is to manage _all_ networking related components to enable execution of RDMA and GPUDirect +RDMA workloads in a kubernetes cluster including: + +* Mellanox Networking drivers to enable advanced features +* Kubernetes device plugins to provide hardware resources for fast network +* Kubernetes secondary network for Network intensive workloads + +### Documentation + +For more information please visit the official [documentation](https://docs.nvidia.com/networking/software/cloud-orchestration/index.html). + +## Additional components + +### Node Feature Discovery + +Nvidia Network Operator relies on the existance of specific node labels to operate properly. e.g label a node as having +Nvidia networking hardware available. This can be achieved by either manually labeling Kubernetes nodes or using +[Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery) to perform the labeling. + +To allow zero touch deployment of the Operator we provide a helm chart to be used to optionally deploy Node Feature +Discovery in the cluster. This is enabled via `nfd.enabled` chart parameter. + +### SR-IOV Network Operator + +Nvidia Network Operator can operate in unison with SR-IOV Network Operator to enable SR-IOV workloads in a Kubernetes +cluster. We provide a helm chart to be used to optionally +deploy [SR-IOV Network Operator](https://github.com/k8snetworkplumbingwg/sriov-network-operator) in the cluster. This is +enabled via `sriovNetworkOperator.enabled` chart parameter. + +SR-IOV Network Operator can work in conjuction with [IB Kubernetes](#ib-kubernetes) to use InfiniBand PKEY Membership +Types + +For more information on how to configure SR-IOV in your Kubernetes cluster using SR-IOV Network Operator refer to the +project's github. + +## QuickStart + +### System Requirements + +* RDMA capable hardware: Mellanox ConnectX-5 NIC or newer. +* NVIDIA GPU and driver supporting GPUDirect e.g Quadro RTX 6000/8000 or Tesla T4 or Tesla V100 or Tesla V100. + (GPU-Direct only) +* Operating Systems: Ubuntu 20.04 LTS. + +> __NOTE__: ConnectX-6 Lx is not supported. + +### Tested Network Adapters + +The following Network Adapters have been tested with network-operator: + +* ConnectX-5 +* ConnectX-6 Dx + +### Prerequisites + +- Kubernetes v1.17+ +- Helm v3.5.3+ +- Ubuntu 20.04 LTS + +### Install Helm + +Helm provides an install script to copy helm binary to your system: + +``` +$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +$ chmod 500 get_helm.sh +$ ./get_helm.sh +``` + +For additional information and methods for installing Helm, refer to the official [helm website](https://helm.sh/) + +### Deploy Network Operator + +``` +# Add Repo +$ helm repo add nvidia https://helm.ngc.nvidia.com/nvidia +$ helm repo update + +# Install Operator +$ helm install -n network-operator --create-namespace --wait network-operator nvidia/network-operator + +# View deployed resources +$ kubectl -n network-operator get pods +``` + +#### Deploy Network Operator without Node Feature Discovery + +By default the network operator +deploys [Node Feature Discovery (NFD)](https://github.com/kubernetes-sigs/node-feature-discovery) +in order to perform node labeling in the cluster to allow proper scheduling of Network Operator resources. If the nodes +where already labeled by other means (either deployed from upstream or deployed within another deployment), it is possible to disable the deployment of NFD by setting +`nfd.enabled=false` chart parameter and make sure that the installed version is `v0.13.2` or newer and has NodeFeatureApi enabled. + +##### Deploy NFD from upstream with NodeFeatureApi enabled +``` +$ export NFD_NS=node-feature-discovery +$ helm repo add nfd https://kubernetes-sigs.github.io/node-feature-discovery/charts +$ helm repo update +$ helm install nfd/node-feature-discovery --namespace $NFD_NS --create-namespace --generate-name --set enableNodeFeatureApi='true' +``` +For additional information , refer to the official [NVD deployment with Helm](https://kubernetes-sigs.github.io/node-feature-discovery/v0.13/deployment/helm.html) + +##### Deploy Network Operator without Node Feature Discovery +``` +$ helm install --set nfd.enabled=false -n network-operator --create-namespace --wait network-operator nvidia/network-operator +``` + +##### Currently the following NFD labels are used: + +| Label | Where | +|-----------------------------------------------|---------------------------------------------------| +| `feature.node.kubernetes.io/pci-15b3.present` | Nodes bearing Nvidia Mellanox Networking hardware | +| `nvidia.com/gpu.present` | Nodes bearing Nvidia GPU hardware | + +> __Note:__ The labels which Network Operator depends on may change between releases. + +> __Note:__ By default the operator is deployed without an instance of `NicClusterPolicy` and `MacvlanNetwork` +> custom resources. The user is required to create it later with configuration matching the cluster or use chart parameters to deploy it together with the operator. + +#### Deploy Network Operator with GitOps + +The chart runs a `keep-ncp` hook by default to preserve a Helm-managed `NicClusterPolicy` when upgrading from an older +chart version that deployed the custom resource. GitOps controllers such as Argo CD cannot distinguish a Helm install +from an upgrade and run this hook during every sync. + +For a first-time GitOps installation where there is no existing Helm-managed `NicClusterPolicy`, disable the hook and +its RBAC resources by setting: + +```yaml +keepNCP: false +``` + +Keep the default value, `keepNCP: true`, when upgrading a deployment that might contain a Helm-managed +`NicClusterPolicy` from an older chart version. + +#### Deploy development version of Network Operator + +To install development version of Network Operator you need to clone repository first and install helm chart from the +local directory: + +``` +# Clone Network Operator Repository +$ git clone https://github.com/Mellanox/network-operator.git + +# Update chart dependencies +$ cd network-operator/deployment/network-operator && helm dependency update + +# Install Operator +$ helm install -n network-operator --create-namespace --wait network-operator ./ + +# View deployed resources +$ kubectl -n network-operator get pods +``` + +#### Deploy Network Operator with Admission Controller + +The Admission Controller can be optionally included as part of the Network Operator installation process. +It has the capability to validate supported Custom Resource Definitions (CRDs), which currently include NicClusterPolicy and HostDeviceNetwork. +By default, the deployment of the admission controller is disabled. To enable it, you must set `operator.admissionController.enabled` to `true`. + +Enabling the admission controller provides you with two options for managing certificates. +You can either utilize [cert-manager](https://cert-manager.io/docs/installation/) for generating a self-signed certificate automatically, or you can provide your own self-signed certificate. + +To use `cert-manager`, ensure that `operator.admissionController.useCertManager` is set to `true`. Additionally, make sure that you deploy cert-manager before initiating the Network Operator deployment. + +If you prefer not to use `cert-manager`, set `operator.admissionController.useCertManager` to `false`, and then provide your custom certificate and key using `operator.admissionController.certificate.tlsCrt` and `operator.admissionController.certificate.tlsKey`. + +> __NOTE__: When using your own certificate, the certificate must be valid for -webhook-service.< +> Release_Namespace>.svc, e.g. network-operator-webhook-service.network-operator.svc + +> __NOTE__: When deploying network operator with admission controller using helm, you need to append `--wait` to helm install and helm upgrade commands +> + +##### Generating self-signed certificate using OpenSSL + +To generate a self-signed SSL certificate valid for a specific hostname, you can use the `openssl` command-line tool. +First, navigate to the directory where you want to store your certificate and key files. Then, run the following +command: + +```bash +SVCNAME="network-operator-webhook-service.network-operator.svc" +openssl req -x509 -nodes -batch -newkey rsa:2048 -keyout server.key -out server.crt -days 365 -addext "subjectAltName=DNS:$SVCNAME" +``` + +Replace `SVCNAME` with the SVC name follows this convention -webhook-service..svc. +This command will generate a new RSA key pair with 2048 bits and create a self-signed certificate (`server.crt`) and +private key (`server.key`) that are valid for 365 days. + +#### Deploy Network Operator with Go integration coverage + +For integration test coverage collection (e.g. Jinx regression), enable coverage and deploy +with the immutable instrumented image tag published by CI (`-coverage`): + +```bash +helm install network-operator ./deployment/network-operator \ + --set operator.coverage.enabled=true \ + --set operator.tag=-coverage +``` + +When enabled, the controller sets `GOCOVERDIR=/coverage`, mounts an `emptyDir` at `/coverage`, and +sets `COVERAGE_CLEAR_AFTER_FLUSH=1`. Send **SIGUSR1** to the manager process to flush runtime +coverage counters without restarting the pod. On distroless images, signal the container host PID +rather than relying on `kubectl exec kill`. + +## Upgrade + +> __NOTE__: Upgrade capabilities are limited now. Additional manual actions required when containerized OFED driver is used + +Before starting the upgrade to a specific release version, please, check release notes for this version to ensure that +no additional actions are required. + + +### Check available releases + +``` +helm search repo nvidia/network-operator -l +``` + +> __NOTE__: add `--devel` option if you want to list beta releases as well + +### Upgrade CRDs to compatible version + +The network-operator helm chart contains a hook(pre-install, pre-upgrade) that will automatically upgrade required CRDs in the cluster. +The hook is enabled by default. If you don't want to upgrade CRDs with helm automatically, +you can disable auto upgrade by setting `upgradeCRDs: false` in the helm chart values. +Then you can follow the guide below to download and apply CRDs for the concrete version of the network-operator. + +It is possible to retrieve updated CRDs from the Helm chart or from the release branch on GitHub. Example bellow show +how to download and unpack Helm chart for specified release and then apply CRDs update from it. + +``` +helm pull nvidia/network-operator --version --untar --untardir network-operator-chart +``` + +> __NOTE__: `--devel` option required if you want to use the beta release + +``` +kubectl apply -f network-operator-chart/network-operator/crds \ + -f network-operator-chart/network-operator/charts/sriov-network-operator/crds +``` + +### Prepare Helm values for the new release + +Download Helm values for the specific release + +``` +helm show values nvidia/network-operator --version= > values-.yaml +``` + +Edit `values-.yaml` file as required for your cluster. + +### Apply Helm chart update + +``` +helm upgrade -n network-operator network-operator nvidia/network-operator --version= -f values-.yaml --force +``` + +> __NOTE__: `--devel` option required if you want to use the beta release + +## Chart parameters + +In order to tailor the deployment of the network operator to your cluster needs, Chart parameters are available. See official [documentation](https://docs.nvidia.com/networking/software/cloud-orchestration/index.html). diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/.helmignore b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/Chart.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/Chart.yaml new file mode 100644 index 00000000..8b73ff4c --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: maintenance-operator-chart +description: Maintenance Operator Helm Chart +type: application +version: 0.0.1 +appVersion: "latest" diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/README.md b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/README.md new file mode 100644 index 00000000..f27c7d4f --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/README.md @@ -0,0 +1,35 @@ +# maintenance-operator-chart + +![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square) + +Maintenance Operator Helm Chart + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| imagePullSecrets | list | `[]` | image pull secrets for the operator | +| metricsService | object | `{"ports":[{"name":"https","port":8443,"protocol":"TCP","targetPort":"https"}],"type":"ClusterIP"}` | metrics service configurations | +| operator.admissionController.certificates.certManager.enable | bool | `true` | use cert-manager for certificates | +| operator.admissionController.certificates.certManager.generateSelfSigned | bool | `true` | generate self-signed certificiates with cert-manager | +| operator.admissionController.certificates.custom.enable | bool | `false` | enable custom certificates using secrets | +| operator.admissionController.certificates.secretNames.operator | string | `"operator-webhook-cert"` | secret name containing certificates for the operator admission controller | +| operator.admissionController.enable | bool | `true` | enable admission controller of the operator | +| operator.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/master","operator":"Exists"}]},"weight":1},{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | node affinity for the operator | +| operator.image.imagePullPolicy | string | `nil` | image pull policy for the operator image | +| operator.image.repository | string | `"ghcr.io/mellanox"` | repository to use for the operator image | +| operator.image.name | string | `"maintenance-operator"` | image name to use for the operator image | +| operator.image.tag | string | `nil` | image tag to use for the operator image | +| operator.nodeSelector | object | `{}` | node selector for the operator | +| operator.replicas | int | `1` | operator deployment number of repplicas | +| operator.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}` | specify resource requests and limits for the operator | +| operator.serviceAccount.annotations | object | `{}` | set annotations for the operator service account | +| operator.tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]` | toleration for the operator | +| operatorConfig | object | `{"logLevel":"info","maxNodeMaintenanceTimeSeconds":null,"maxParallelOperations":null,"maxUnavailable":null}` | operator configuration values. fields here correspond to fields in MaintenanceOperatorConfig CR | +| operatorConfig.deploy | bool | `false` | deploy operatorConfig custom resource | +| operatorConfig.logLevel | string | `"info"` | log level configuration | +| operatorConfig.maxNodeMaintenanceTimeSeconds | string | `nil` | max time for node maintenance | +| operatorConfig.maxParallelOperations | string | `nil` | max number of parallel operations | +| operatorConfig.maxUnavailable | string | `nil` | max number of unavailable nodes | +| webhookService | object | `{"ports":[{"port":443,"protocol":"TCP","targetPort":9443}],"type":"ClusterIP"}` | webhook service configurations | + diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml new file mode 100644 index 00000000..23a03d79 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml @@ -0,0 +1,93 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: maintenanceoperatorconfigs.maintenance.nvidia.com +spec: + group: maintenance.nvidia.com + names: + kind: MaintenanceOperatorConfig + listKind: MaintenanceOperatorConfigList + plural: maintenanceoperatorconfigs + singular: maintenanceoperatorconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: MaintenanceOperatorConfig is the Schema for the maintenanceoperatorconfigs + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: MaintenanceOperatorConfigSpec defines the desired state of + MaintenanceOperatorConfig + properties: + logLevel: + default: info + description: LogLevel is the operator logging level + enum: + - debug + - info + - error + type: string + maxNodeMaintenanceTimeSeconds: + default: 3600 + description: |- + MaxNodeMaintenanceTimeSeconds is the time from when a NodeMaintenance is marked as ready (phase: Ready) + until the NodeMaintenance is considered stale and removed by the operator. + should be less than idle time for any autoscaler that is running. + default to 60m (3600 seconds) + format: int32 + minimum: 0 + type: integer + maxParallelOperations: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + MaxParallelOperations indicates the maximal number nodes that can undergo maintenance + at a given time. 0 means no limit + value can be an absolute number (ex: 5) or a percentage of total nodes in the cluster (ex: 10%). + absolute number is calculated from percentage by rounding up. + defaults to 1. The actual number of nodes that can undergo maintenance may be lower depending + on the value of MaintenanceOperatorConfigSpec.MaxUnavailable. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of nodes that can become unavailable in the cluster. + value can be an absolute number (ex: 5) or a percentage of total nodes in the cluster (ex: 10%). + absolute number is calculated from percentage by rounding up. + by default, unset. + new nodes will not be processed if the number of unavailable node will exceed this value + x-kubernetes-int-or-string: true + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml new file mode 100644 index 00000000..b78e5840 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml @@ -0,0 +1,275 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: nodemaintenances.maintenance.nvidia.com +spec: + group: maintenance.nvidia.com + names: + kind: NodeMaintenance + listKind: NodeMaintenanceList + plural: nodemaintenances + singular: nodemaintenance + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.nodeName + name: Node + type: string + - jsonPath: .spec.requestorID + name: Requestor + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Phase + type: string + - jsonPath: .status.conditions[?(@.type=='Failed')].reason + name: Failed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeMaintenance is the Schema for the nodemaintenances API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeMaintenanceSpec defines the desired state of NodeMaintenance + properties: + additionalRequestors: + description: |- + AdditionalRequestors is a set of additional requestor IDs which are using the same NodeMaintenance + request. addition or removal of requiestor IDs to this list MUST be made with update operation (and retry on failure) + which will replace the entire list. + items: + type: string + type: array + x-kubernetes-list-type: set + cordon: + default: true + description: Cordon if set, marks node as unschedulable during maintenance + operation + type: boolean + drainSpec: + description: DrainSpec specifies how a node will be drained. if not + provided, no draining will be performed. + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the node is drained) + type: boolean + force: + default: false + description: Force draining even if there are pods that do not + declare a controller + type: boolean + podEvictionFilters: + description: |- + PodEvictionFilters specifies filters for pods that need to undergo eviction during drain. + if specified. only pods that match PodEvictionFilters will be evicted during drain operation. + if unspecified. all non-daemonset pods will be evicted. + logical OR is performed between filter entires. logical AND is performed within different filters + in a filter entry. + items: + description: PodEvictionFiterEntry defines filters for Pod evictions + during drain operation + properties: + byResourceNameRegex: + description: ByResourceNameRegex filters pods by the name + of the resources they consume using regex. + type: string + type: object + type: array + podSelector: + description: |- + PodSelector specifies a label selector to filter pods on the node that need to be drained + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 300 + description: TimeoutSecond specifies the length of time in seconds + to wait before giving up drain, zero means infinite + format: int32 + minimum: 0 + type: integer + type: object + nodeName: + description: |- + NodeName is The name of the node that maintenance operation will be performed on + creation fails if node obj does not exist (webhook) + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + requestorID: + description: |- + RequestorID MUST follow domain name notation format (https://tools.ietf.org/html/rfc1035#section-2.3.1) + It MUST be 63 characters or less, beginning and ending with an alphanumeric + character ([a-z0-9A-Z]) with dashes (-), dots (.), and alphanumerics between. + caller SHOULD NOT create multiple objects with same requestorID and nodeName. + This field identifies the requestor of the operation. + maxLength: 63 + minLength: 2 + pattern: ^([a-z0-9A-Z]([-a-z0-9A-Z]*[a-z0-9A-Z])?(\.[a-z0-9A-Z]([-a-z0-9A-Z]*[a-z0-9A-Z])?)*)$ + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + waitForPodCompletion: + description: |- + WaitForPodCompletion specifies pods via selector to wait for completion before performing drain operation + if not provided, will not wait for pods to complete + properties: + podSelector: + description: |- + PodSelector specifies a label selector for the pods to wait for completion + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + example: app=my-workloads + type: string + timeoutSeconds: + default: 0 + description: |- + TimeoutSecond specifies the length of time in seconds + to wait before giving up on pod termination, zero means infinite + format: int32 + minimum: 0 + type: integer + type: object + required: + - nodeName + - requestorID + type: object + status: + description: NodeMaintenanceStatus defines the observed state of NodeMaintenance + properties: + conditions: + description: Conditions represents observations of NodeMaintenance + current state + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + drain: + description: Drain represents the drain status of the node + properties: + drainProgress: + description: DrainProgress represents the draining progress as + percentage + format: int32 + minimum: 0 + type: integer + evictionPods: + description: EvictionPods is the total number of pods that need + to be evicted at the time NodeMaintenance started draining + format: int32 + minimum: 0 + type: integer + totalPods: + description: TotalPods is the number of pods on the node at the + time NodeMaintenance started draining + format: int32 + minimum: 0 + type: integer + waitForEviction: + description: WaitForEviction is the list of namespaced named pods + that need to be evicted + items: + type: string + type: array + required: + - drainProgress + - evictionPods + - totalPods + type: object + waitForCompletion: + description: WaitForCompletion is the list of namespaced named pods + that we wait to complete + items: + type: string + type: array + type: object + type: object + selectableFields: + - jsonPath: .spec.nodeName + - jsonPath: .spec.requestorID + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl new file mode 100644 index 00000000..ae44554c --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "maintenance-operator.name" -}} +{{- default "maintenance-operator" .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "maintenance-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default "maintenance-operator" .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "maintenance-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "maintenance-operator.labels" -}} +helm.sh/chart: {{ include "maintenance-operator.chart" . }} +{{ include "maintenance-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "maintenance-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "maintenance-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "maintenance-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "maintenance-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml new file mode 100644 index 00000000..caee32a7 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.operator.admissionController.enable }} +{{- if and .Values.operator.admissionController.certificates.certManager.enable + .Values.operator.admissionController.certificates.certManager.generateSelfSigned }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "maintenance-operator.fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "maintenance-operator.fullname" . }}-serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + dnsNames: + - '{{ include "maintenance-operator.fullname" . }}-webhook-service.{{ .Release.Namespace + }}.svc' + - '{{ include "maintenance-operator.fullname" . }}-webhook-service.{{ .Release.Namespace + }}.svc.{{ .Values.kubernetesClusterDomain | default "cluster.local" }}' + issuerRef: + kind: Issuer + name: '{{ include "maintenance-operator.fullname" . }}-selfsigned-issuer' + secretName: {{ .Values.operator.admissionController.certificates.secretNames.operator }} +{{- else if and (not .Values.operator.admissionController.certManager.enable) .Values.operator.admissionController.custom.enable }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.operator.admissionController.certificates.secretNames.operator }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.operator.admissionController.certificates.custom.operator.caCrt | b64enc | b64enc | quote }} + tls.crt: {{ .Values.operator.admissionController.certificates.custom.operator.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionController.certificates.custom.operator.tlsKey | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml new file mode 100644 index 00000000..134b720e --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "maintenance-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: maintenance-operator-controller-manager + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.operator.replicas }} + selector: + matchLabels: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + control-plane: {{ .Release.Name }}-controller-manager + app.kubernetes.io/component: maintenance-operator-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: manager + spec: + priorityClassName: system-cluster-critical + tolerations: {{- toYaml .Values.operator.tolerations | nindent 8 }} + nodeSelector: {{- toYaml .Values.operator.nodeSelector | nindent 8 }} + affinity: {{- toYaml .Values.operator.affinity | nindent 8 }} + imagePullSecrets: {{ .Values.imagePullSecrets | default list | toJson }} + securityContext: + runAsNonRoot: true + serviceAccountName: {{ include "maintenance-operator.fullname" . }}-controller-manager + terminationGracePeriodSeconds: 10 + containers: + - name: manager + command: + - /manager + args: + - --leader-elect + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_WEBHOOKS + value: {{ quote .Values.operator.admissionController.enable }} + image: {{ .Values.operator.image.repository }}/{{ .Values.operator.image.name }}:{{ .Values.operator.image.tag | default .Chart.AppVersion }} + {{- if .Values.operator.image.imagePullPolicy }} + imagePullPolicy: {{ .Values.operator.image.imagePullPolicy }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + {{- if .Values.operator.admissionController.enable }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + {{- end }} + resources: {{- toYaml .Values.operator.resources | nindent 10 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + {{- if .Values.operator.admissionController.enable }} + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + volumes: + {{- if .Values.operator.admissionController.enable }} + - name: cert + secret: + defaultMode: 420 + secretName: {{ .Values.operator.admissionController.certificates.secretNames.operator }} + {{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml new file mode 100644 index 00000000..fb97ffad --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml @@ -0,0 +1,23 @@ +{{- if .Values.operatorConfig.deploy }} +apiVersion: maintenance.nvidia.com/v1alpha1 +kind: MaintenanceOperatorConfig +metadata: + name: default + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: config + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + logLevel: {{ .Values.operatorConfig.logLevel }} +{{- if .Values.operatorConfig.maxParallelOperations }} + maxParallelOperations: {{ .Values.operatorConfig.maxParallelOperations }} +{{- end }} +{{- if .Values.operatorConfig.maxUnavailable }} + maxUnavailable: {{ .Values.operatorConfig.maxUnavailable }} +{{- end }} +{{- if .Values.operatorConfig.maxNodeMaintenanceTimeSeconds }} + maxNodeMaintenanceTimeSeconds: {{ .Values.operatorConfig.maxNodeMaintenanceTimeSeconds }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role.yaml new file mode 100644 index 00000000..464e1a15 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role.yaml @@ -0,0 +1,154 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-role + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + - delete + - get + - list + - patch + - update +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + - watch +- apiGroups: + - config.openshift.io + resources: + - infrastructures + verbs: + - get + - list + - watch +- apiGroups: + - machineconfiguration.openshift.io + resources: + - machineconfigpools + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - machineconfiguration.openshift.io + resources: + - machineconfigs + verbs: + - get + - list + - watch +- apiGroups: + - maintenance.nvidia.com + resources: + - maintenanceoperatorconfigs + - nodemaintenances + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - maintenance.nvidia.com + resources: + - maintenanceoperatorconfigs/finalizers + - nodemaintenances/finalizers + verbs: + - update +- apiGroups: + - maintenance.nvidia.com + resources: + - maintenanceoperatorconfigs/status + - nodemaintenances/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-role + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml new file mode 100644 index 00000000..6f3491d9 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-rolebinding + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: '{{ include "maintenance-operator.fullname" . }}-manager-role' +subjects: +- kind: ServiceAccount + name: '{{ include "maintenance-operator.fullname" . }}-controller-manager' + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-rolebinding + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: '{{ include "maintenance-operator.fullname" . }}-manager-role' +subjects: +- kind: ServiceAccount + name: '{{ include "maintenance-operator.fullname" . }}-controller-manager' + namespace: {{ .Release.Namespace }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml new file mode 100644 index 00000000..c948f0c8 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "maintenance-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.operator.serviceAccount.annotations | nindent 4 }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml new file mode 100644 index 00000000..5e904c68 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml @@ -0,0 +1,48 @@ +{{- if .Values.operator.admissionController.enable }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "maintenance-operator.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + type: {{ .Values.webhookService.type }} + selector: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 4 }} + ports: + {{- .Values.webhookService.ports | toYaml | nindent 2 }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "maintenance-operator.fullname" . }}-validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "maintenance-operator.fullname" . }}-serving-cert + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ include "maintenance-operator.fullname" . }}-webhook-service' + namespace: {{ .Release.Namespace }} + path: /validate-maintenance-nvidia-com-v1alpha1-nodemaintenance + failurePolicy: Fail + name: vnodemaintenance.kb.io + rules: + - apiGroups: + - maintenance.nvidia.com + apiVersions: + - v1alpha1 + operations: + - CREATE + resources: + - nodemaintenances + sideEffects: None +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/values.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/values.yaml new file mode 100644 index 00000000..3848c146 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/values.yaml @@ -0,0 +1,114 @@ +operator: + image: + # -- repository to use for the operator image + repository: ghcr.io/mellanox + # -- image name to use for the operator image + name: maintenance-operator + # -- image tag to use for the operator image + tag: null + # -- image pull policy for the operator image + imagePullPolicy: null + # -- toleration for the operator + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + # -- node selector for the operator + nodeSelector: {} + # -- node affinity for the operator + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + # -- specify resource requests and limits for the operator + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + # -- operator deployment number of repplicas + replicas: 1 + serviceAccount: + # -- set annotations for the operator service account + annotations: {} + admissionController: + # -- enable admission controller of the operator + enable: true + certificates: + secretNames: + # -- secret name containing certificates for the operator admission controller + operator: "operator-webhook-cert" + certManager: + # -- use cert-manager for certificates + enable: true + # -- generate self-signed certificiates with cert-manager + generateSelfSigned: true + custom: + # -- enable custom certificates using secrets + enable: false + # operator: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + +# -- operator configuration values. fields here correspond to fields in MaintenanceOperatorConfig CR +operatorConfig: + # -- deploy operatorConfig CR with the below values + deploy: false + # -- log level configuration + logLevel: info + # operatorConfig.maxParallelOperations -- max number of parallel operations + maxParallelOperations: null + # -- max number of unavailable nodes + maxUnavailable: null + # -- max time for node maintenance + maxNodeMaintenanceTimeSeconds: null + +# -- image pull secrets for the operator +imagePullSecrets: [] + +# -- metrics service configurations +metricsService: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + type: ClusterIP + +# -- webhook service configurations +webhookService: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + type: ClusterIP + +# -- Kubernetes cluster domain used for webhook certificate DNS names (defaults to cluster.local if not set) +# kubernetesClusterDomain: cluster.local diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/.helmignore b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/Chart.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/Chart.yaml new file mode 100644 index 00000000..db11d777 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v2 +appVersion: master +icon: https://kubernetes-sigs.github.io/node-feature-discovery/master/assets/images/nfd/favicon.svg +description: | + Node Feature Discovery (NFD) is a Kubernetes add-on for detecting hardware + features and system configuration. Detected features are advertised as node + labels. NFD provides flexible configuration and extension points for a wide + range of vendor and application specific node labeling needs. +name: node-feature-discovery +sources: + - https://github.com/kubernetes-sigs/node-feature-discovery +home: https://github.com/kubernetes-sigs/node-feature-discovery +keywords: + - feature-discovery + - feature-detection + - node-labels +type: application +version: 0.2.1 diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md new file mode 100644 index 00000000..93d6edf6 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md @@ -0,0 +1,394 @@ +# node-feature-discovery + +![Version: 0.2.1](https://img.shields.io/badge/Version-0.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: master](https://img.shields.io/badge/AppVersion-master-informational?style=flat-square) +Node Feature Discovery (NFD) is a Kubernetes add-on for detecting hardware +features and system configuration. Detected features are advertised as node +labels. NFD provides flexible configuration and extension points for a wide +range of vendor and application specific node labeling needs. + +**Homepage:** + +## Source Code + +* + +## Installing the chart + +### OCI Helm repository + +The NFD project provides Helm charts in an OCI compliant repository. Install +NFD with the default configuration + +```bash +helm install nfd --namespace node-feature-discovery --create-namespace oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master +``` + +See the [configuration](#configuration) section below for instructions how to +alter the deployment parameters. + +### Legacy Helm repository + +For the time being, the NFD project still provides Helm charts in a legacy +(HTTP) Helm repository, too. + +> **NOTE:** This repository will be deprecated in the future. It is recommended +> for users to switch to use the OCI Helm repository. + +#### Stable version + +To install the latest stable version: + +```bash +helm repo add nfd https://kubernetes-sigs.github.io/node-feature-discovery/charts +helm repo update +helm install nfd nfd/node-feature-discovery --namespace node-feature-discovery --create-namespace +``` + +See the [configuration](#configuration) section below for instructions how to +alter the deployment parameters. + +### Latest development version + +To install the latest development version you need to clone the NFD Git +repository and install from there. + +```bash +git clone https://github.com/kubernetes-sigs/node-feature-discovery/ +cd node-feature-discovery/deployment/helm +helm install nfd ./node-feature-discovery/ --namespace node-feature-discovery --create-namespace +``` + +## Configuration + +You can override values from `values.yaml` and provide a file with custom values: + +```bash +helm install -f --namespace nfd --create-namespace nfd oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master +``` + +To specify each parameter separately you can provide them to helm install command: + +```bash +helm install --set nameOverride=NFDinstance --set master.replicaCount=2 --namespace nfd --create-namespace nfd oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master +``` + +## Upgrading the chart + +To upgrade the `node-feature-discovery` deployment to master via Helm. + +### Rolling-update pace on large clusters + +The nfd-worker DaemonSet rolls out updates with `maxUnavailable: "10%"` by +default, so an upgrade completes in roughly ten waves regardless of cluster +size. nfd-worker is stateless and node labels persist while a worker pod +restarts, so tuning the pace only affects how quickly feature updates resume +on each node. Adjust it via `worker.updateStrategy.rollingUpdate.maxUnavailable` +(e.g. `1` restores the Kubernetes default of one node at a time; to use +`type: OnDelete`, also set `rollingUpdate: null` as Helm deep-merges maps). + +When driving upgrades through `helm upgrade --wait` or GitOps health checks +(Flux `HelmRelease`, Argo CD), size the timeout to cover the full rollout: +roughly `ceil(1 / maxUnavailable) × per-wave pod-ready time`. With the serial +Kubernetes default the rollout time grows linearly with node count and easily +exceeds a 5-minute timeout on large clusters. + +### From v0.7 and older + +Please see +the [uninstallation guide](https://kubernetes-sigs.github.io/node-feature-discovery/v0.7/get-started/deployment-and-usage.html#uninstallation). +And then follow the standard [installation instructions](#installing-the-chart). + +### From v0.8 - v0.11 + +Helm deployment of NFD was introduced in v0.8.0. + +```bash +export NFD_NS=node-feature-discovery +export HELM_INSTALL_NAME=nfd +# Uninstall the old NFD deployment +helm uninstall $HELM_INSTALL_NAME --namespace $NFD_NS +# Install the new NFD deployment +helm install $HELM_INSTALL_NAME oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master --namespace $NFD_NS --set master.enable=false +# Wait for NFD Worker to be ready +kubectl wait --timeout=-1s --for=condition=ready pod -l app.kubernetes.io/name=node-feature-discovery --namespace $NFD_NS +# Enable the NFD Master +helm upgrade $HELM_INSTALL_NAME oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master --namespace $NFD_NS --set master.enable=true +``` + +### From v0.12 - v0.13 + +In v0.12 the `NodeFeature` CRD was introduced as experimental. +The API was not enabled by default. + +```bash +export NFD_NS=node-feature-discovery +export HELM_INSTALL_NAME=nfd +# Install and upgrade CRD's +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/node-feature-discovery/master/deployment/base/nfd-crds/nfd-api-crds.yaml +# Install the new NFD deployment +helm upgrade $HELM_INSTALL_NAME oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master --namespace $NFD_NS --set master.enable=false +# Wait for NFD Worker to be ready +kubectl wait --timeout=-1s --for=condition=ready pod -l app.kubernetes.io/name=node-feature-discovery --namespace $NFD_NS +# Enable the NFD Master +helm upgrade $HELM_INSTALL_NAME oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master --namespace $NFD_NS --set master.enable=true +``` + +### From v0.14+ + +As of version v0.14 the Helm chart is the primary deployment method for NFD, +and the CRD `NodeFeature` is enabled by default. + +```bash +export NFD_NS=node-feature-discovery +export HELM_INSTALL_NAME=nfd +# Install and upgrade CRD's +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/node-feature-discovery/master/deployment/base/nfd-crds/nfd-api-crds.yaml +# Install the new NFD deployment +helm upgrade $HELM_INSTALL_NAME oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery --version 0.0.0-master --namespace $NFD_NS +``` + +## Uninstalling the chart + +To uninstall the `node-feature-discovery` deployment: + +```bash +helm uninstall nfd --namespace node-feature-discovery +``` + +The command removes all the Kubernetes components associated with the chart and +deletes the release. It also runs a post-delete hook that cleans up the nodes +of all labels, annotations, taints and extended resources that were created by +NFD. + +## Values + +### General + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| image.repository | string | `"gcr.io/k8s-staging-nfd/node-feature-discovery"` | NFD image repository | +| image.pullPolicy | string | `"Always"` | Image pull policy | +| image.tag | string | `nil` | NFD image tag. If not specified Chart.AppVersion will be used. | +| imagePullSecrets | list | `[]` | Image pull secrets. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod). | +| nameOverride | string | `""` | Override the name of the chart | +| fullnameOverride | string | `""` | Override a default fully qualified app name | +| namespaceOverride | string | `""` | Override the namespace to install the chart into. By default, the namespace is determined by Helm. | +| featureGates | object | `{}` | [Feature gates](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/feature-gates) to enable/disable specific features. | +| checksumValuesOnly | bool | `false` | Only checksum config values (not the full rendered ConfigMap template) for the rollout annotation. When enabled, pods will only restart when config values actually change, not on unrelated template changes. | +| priorityClassName | string | `""` | The name of the PriorityClass to be used for the NFD pods. | +| postDeleteCleanup | bool | `true` | Enable/disable the Helm post-delete hook. | + +### Global + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| global.imagePullSecrets | list | `[]` | If `imagePullSecrets` is specified, it takes precedence over `global.imagePullSecrets`. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod). | + +### NFD-Master + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| master.enable | bool | `true` | Specifies whether nfd-master should be deployed | +| master.extraArgs | list | `[]` | Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/master-commandline-reference) to pass to nfd-master. | +| master.extraEnvs | list | `[]` | Additional environment variables to set in the nfd-master container. | +| master.hostNetwork | bool | `false` | Run the container in the host's network namespace. | +| master.hostUsers | bool | `nil` | Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. | +| master.dnsPolicy | string | `"ClusterFirstWithHostNet"` | NFD master pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy). | +| master.config | string | `nil` | NFD master [configuration](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/master-configuration-reference). | +| master.port | int | `8080` | Port on which to serve http for metrics and healthz endpoints. | +| master.instance | string | `nil` | Instance name. Used to separate annotation namespaces for multiple parallel deployments. | +| master.resyncPeriod | int | `nil` | NFD API controller resync period. Time duration string (e.g. "5m", "1h", "2h45m"). | +| master.denyLabelNs | list | `[]` | Label namespaces to deny. Labels with these prefixes will not be published to the nodes. | +| master.extraLabelNs | list | `[]` | Additional label namespaces to publish. Labels with these prefixes are allowed even if otherwise denied by `master.denyLabelNs`. | +| master.enableTaints | bool | `false` | Enable node tainting. | +| master.nfdApiParallelism | int | `nil` | The maximum number of concurrent node updates. | +| master.deploymentAnnotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-master Deployment. | +| master.replicaCount | int | `1` | Number of the desired replicas for the nfd-master Deployment. | +| master.podSecurityContext | object | `{}` | [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-master pods. | +| master.initContainers | list | `[]` | [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-master pods. | +| master.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-master container. | +| master.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | +| master.serviceAccount.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account | +| master.serviceAccount.name | string | `nil` | The name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| master.revisionHistoryLimit | int | `nil` | Specifies the number of old ReplicaSets for the Deployment to retain. [revisionHistoryLimit](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#revision-history-limit) | +| master.rbac.create | bool | `true` | Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-master. | +| master.resources.limits | object | `{"memory":"4Gi"}` | Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-master container. | +| master.resources.requests | object | `{"cpu":"100m","memory":"128Mi"}` | Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-master container. | +| master.nodeSelector | object | `{}` | [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-master pods. | +| master.tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Equal","value":""}]` | [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-master pods. | +| master.podDisruptionBudget.enable | bool | `false` | Configure PodDisruptionBudget for the nfd-master Deployment. | +| master.podDisruptionBudget.minAvailable | int | `1` | Minimum number (or percentage) of pods that must be available after the eviction. | +| master.podDisruptionBudget.unhealthyPodEvictionPolicy | string | `"AlwaysAllow"` | Policy to evict unhealthy pods when a PodDisruptionBudget is defined. | +| master.networkPolicy.enabled | bool | `false` | Should a networkPolicy be deployed for the nfd-master pods | +| master.networkPolicy.egress | list | `[{"ports":[{"port":80,"protocol":"TCP"},{"port":443,"protocol":"TCP"},{"port":53,"protocol":"TCP"},{"port":53,"protocol":"UDP"},{"port":6443,"protocol":"TCP"}]}]` | [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-master pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp | +| master.networkPolicy.ingress | list | `[{"ports":[{"port":"http","protocol":"TCP"}]}]` | [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-master pods. | +| master.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-master pods. | +| master.labels | object | `{}` | [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-master pods. | +| master.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"In","values":[""]}]},"weight":1}]}}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-master pods. | +| master.startupProbe | object | - | Startup probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-startup-probes). | +| master.startupProbe.initialDelaySeconds | int | `nil` | The number of seconds after the container has started before probe is initiated. | +| master.startupProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| master.startupProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| master.startupProbe.failureThreshold | int | `30` | The number of consecutive failures for the probe before considering the pod as not ready. | +| master.livenessProbe | object | - | Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). | +| master.livenessProbe.initialDelaySeconds | int | `nil` | The number of seconds after the container has started before probe is initiated. | +| master.livenessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| master.livenessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| master.livenessProbe.failureThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| master.readinessProbe | object | - | Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). | +| master.readinessProbe.initialDelaySeconds | int | `nil` | The number of seconds after the container has started before probe is initiated. | +| master.readinessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| master.readinessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| master.readinessProbe.successThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| master.readinessProbe.failureThreshold | int | `10` | The number of consecutive failures for the probe before considering the pod as not ready. | + +### NFD-Worker + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| worker.enable | bool | `true` | Specifies whether nfd-worker should be deployed | +| worker.ownerRefs | list | `["pod","ds"]` | Objects used as owner references for NodeFeature objects. Valid values are `node`, `pod`, and `ds`. This value is passed through `-owner-refs` and takes precedence over `worker.config.core.ownerRefs`. | +| worker.extraArgs | list | `[]` | Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/worker-commandline-reference) to pass to nfd-worker. | +| worker.extraEnvs | list | `[]` | Additional environment variables to set in the nfd-worker container. | +| worker.hostNetwork | bool | `false` | Run the container in the host's network namespace. | +| worker.hostUsers | bool | `nil` | Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. | +| worker.dnsPolicy | string | `"ClusterFirstWithHostNet"` | NFD worker pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy). | +| worker.config | string | `nil` | NFD worker [configuration](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/worker-configuration-reference). | +| worker.port | int | `8080` | Port on which to serve http for metrics and healthz endpoints. | +| worker.daemonsetAnnotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-worker DaemonSet. | +| worker.podSecurityContext | object | `{}` | [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-worker pods. | +| worker.initContainers | list | `[]` | [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-worker pods. | +| worker.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-worker container. | +| worker.livenessProbe | object | - | Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). | +| worker.livenessProbe.initialDelaySeconds | int | `10` | The number of seconds after the container has started before probe is initiated. | +| worker.livenessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| worker.livenessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| worker.livenessProbe.failureThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| worker.readinessProbe | object | - | Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). | +| worker.readinessProbe.initialDelaySeconds | int | `5` | The number of seconds after the container has started before probe is initiated. | +| worker.readinessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| worker.readinessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| worker.readinessProbe.successThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| worker.readinessProbe.failureThreshold | int | `10` | The number of consecutive failures for the probe before considering the pod as not ready. | +| worker.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | +| worker.serviceAccount.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account | +| worker.serviceAccount.name | string | `nil` | Name of the service account to use. If not set and create is true, a name is generated using the fullname template | +| worker.revisionHistoryLimit | int | `nil` | Specifies the number of old history for the DaemonSet to retain to allow rollback. | +| worker.rbac.create | bool | `true` | Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-worker. | +| worker.mountUsrSrc | bool | `false` | Mount host path /user/src inside the container. Does not work on systems without /usr/src AND a read-only /usr. | +| worker.resources.limits | object | `{"memory":"512Mi"}` | Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-worker container. | +| worker.resources.requests | object | `{"cpu":"5m","memory":"64Mi"}` | Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-worker container. | +| worker.nodeSelector | object | `{}` | [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-worker pods. | +| worker.tolerations | list | `[]` | [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-worker pods. | +| worker.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-worker pods. | +| worker.labels | object | `{}` | [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-worker pods. | +| worker.affinity | object | `{}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-worker pods. | +| worker.priorityClassName | string | `nil` | The name of the PriorityClass to be used for the nfd-worker pods. | +| worker.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":"10%"}}` | Update strategy for the nfd-worker DaemonSet. Defaults to a rolling update with `maxUnavailable: "10%"` so upgrades complete in a bounded number of waves on clusters of any size (the Kubernetes default of `maxUnavailable: 1` rolls one node at a time and makes Helm wait and Flux HelmRelease timeouts likely on large clusters). nfd-worker is stateless and node labels persist while a worker pod restarts, so a faster roll is safe. Set `maxUnavailable: 1` to restore the Kubernetes default. To use `type: OnDelete`, also set `rollingUpdate: null` (Helm deep-merges maps). [More info](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set) | +| worker.networkPolicy.enabled | bool | `false` | Should a networkPolicy be deployed for the nfd-worker pods | +| worker.networkPolicy.egress | list | `[{"ports":[{"port":80,"protocol":"TCP"},{"port":443,"protocol":"TCP"},{"port":53,"protocol":"TCP"},{"port":53,"protocol":"UDP"},{"port":6443,"protocol":"TCP"}]}]` | [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-worker pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp | +| worker.networkPolicy.ingress | list | `[{"ports":[{"port":"http","protocol":"TCP"}]}]` | [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-worker pods. | + +### NFD-Topology-Updater + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| topologyUpdater.config | string | `nil` | Configuration for the topology updater. See the [configuration reference](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/topology-updater-configuration-reference) for details. | +| topologyUpdater.enable | bool | `false` | Specifies whether nfd-topology-updater should be deployed. | +| topologyUpdater.createCRDs | bool | `false` | Create the NodeResourceTopology CRD. This MUST be set to true when 'enable' is true, unless the CRD is installed separately (e.g., by another Helm release or external tool). If the CRD is missing, the topology-updater pods will fail with "NodeResourceTopology CRD is not installed" error. | +| topologyUpdater.extraArgs | list | `[]` | Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/topology-updater-commandline-reference) to pass to nfd-topology-updater. | +| topologyUpdater.extraEnvs | list | `[]` | Additional environment variables to set in the nfd-topology-updater container. | +| topologyUpdater.hostNetwork | bool | `false` | Run the container in the host's network namespace. | +| topologyUpdater.hostUsers | bool | `nil` | Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. | +| topologyUpdater.dnsPolicy | string | `"ClusterFirstWithHostNet"` | NFD topology updater pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy). | +| topologyUpdater.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | +| topologyUpdater.serviceAccount.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account. | +| topologyUpdater.serviceAccount.name | string | `nil` | Name or the service account to use. If not set and create is true, a name is generated using the fullname template. | +| topologyUpdater.revisionHistoryLimit | int | `nil` | Specifies the number of old history for the DaemonSet to retain to allow rollback. | +| topologyUpdater.rbac.create | bool | `true` | Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-topology-updater. | +| topologyUpdater.port | int | `8080` | Port on which to serve http for metrics and healthz endpoints. | +| topologyUpdater.kubeletConfigPath | string | `nil` | Host path for the kubelet config file. | +| topologyUpdater.kubeletPodResourcesSockPath | string | `nil` | Host path for the kubelet socket for podresources endpoint. | +| topologyUpdater.updateInterval | string | `"60s"` | Time to sleep between CR updates. Non-positive value implies no CR update. | +| topologyUpdater.watchNamespace | string | `"*"` | Namespace to watch pods, `*` for all namespaces. | +| topologyUpdater.kubeletStateDir | string | `"/var/lib/kubelet"` | The kubelet state directory path for watching state and checkpoint files. Empty value disables kubelet state tracking. | +| topologyUpdater.podSecurityContext | object | `{}` | [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-topology-updater pods. | +| topologyUpdater.initContainers | list | `[]` | [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-topology-updater pods. | +| topologyUpdater.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":true,"runAsUser":0}` | [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-topology-updater container. | +| topologyUpdater.livenessProbe | object | - | Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). | +| topologyUpdater.livenessProbe.initialDelaySeconds | int | `10` | The number of seconds after the container has started before probe is initiated. | +| topologyUpdater.livenessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| topologyUpdater.livenessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| topologyUpdater.livenessProbe.failureThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| topologyUpdater.readinessProbe | object | - | Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). | +| topologyUpdater.readinessProbe.initialDelaySeconds | int | `5` | The number of seconds after the container has started before probe is initiated. | +| topologyUpdater.readinessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| topologyUpdater.readinessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| topologyUpdater.readinessProbe.successThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| topologyUpdater.readinessProbe.failureThreshold | int | `10` | The number of consecutive failures for the probe before considering the pod as not ready. | +| topologyUpdater.resources.limits | object | `{"memory":"60Mi"}` | Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-topology-updater container. | +| topologyUpdater.resources.requests | object | `{"cpu":"50m","memory":"40Mi"}` | Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-topology-updater container. | +| topologyUpdater.nodeSelector | object | `{}` | [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-topology-updater pods. | +| topologyUpdater.tolerations | list | `[]` | [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-topology-updater pods. | +| topologyUpdater.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-topology-updater pods. | +| topologyUpdater.labels | object | `{}` | [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-topology-updater pods. | +| topologyUpdater.daemonsetAnnotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-topology-updater DaemonSet. | +| topologyUpdater.affinity | object | `{}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-topology-updater pods. | +| topologyUpdater.podSetFingerprint | bool | `true` | Enables compute and report of pod fingerprint in NRT objects. | +| topologyUpdater.networkPolicy.enabled | bool | `false` | Should a networkPolicy be deployed for the nfd-topology pods | +| topologyUpdater.networkPolicy.egress | list | `[{"ports":[{"port":80,"protocol":"TCP"},{"port":443,"protocol":"TCP"},{"port":53,"protocol":"TCP"},{"port":53,"protocol":"UDP"},{"port":6443,"protocol":"TCP"}]}]` | [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-topology pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp | +| topologyUpdater.networkPolicy.ingress | list | `[{"ports":[{"port":"http","protocol":"TCP"}]}]` | [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-topology pods. | + +### NFD-GC + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| gc.enable | bool | `true` | Specifies whether nfd-gc should be deployed. | +| gc.extraArgs | list | `[]` | Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/gc-commandline-reference) to pass to nfd-gc. | +| gc.extraEnvs | list | `[]` | Additional environment variables to set in the nfd-gc container. | +| gc.hostNetwork | bool | `false` | Run the container in the host's network namespace. | +| gc.hostUsers | bool | `nil` | Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. | +| gc.replicaCount | int | `1` | The number of desired replicas for the nfd-gc Deployment. | +| gc.dnsPolicy | string | `"ClusterFirstWithHostNet"` | NFD gc pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy). | +| gc.serviceAccount.create | bool | `true` | Specifies whether a service account should be created. | +| gc.serviceAccount.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account. | +| gc.serviceAccount.name | string | `nil` | Name of the service account to use. If not set and create is true, a name is generated using the fullname template. | +| gc.rbac.create | bool | `true` | Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-gc. | +| gc.interval | string | `"1h"` | Time between periodic garbage collector runs. | +| gc.podSecurityContext | object | `{}` | [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-gc pods. | +| gc.initContainers | list | `[]` | [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-gc pods. | +| gc.livenessProbe | object | - | Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). | +| gc.livenessProbe.initialDelaySeconds | int | `10` | The number of seconds after the container has started before probe is initiated. | +| gc.livenessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| gc.livenessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| gc.livenessProbe.failureThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| gc.readinessProbe | object | - | Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). | +| gc.readinessProbe.initialDelaySeconds | int | `5` | The number of seconds after the container has started before probe is initiated. | +| gc.readinessProbe.timeoutSeconds | int | `nil` | The number of seconds after which the probe times out. | +| gc.readinessProbe.periodSeconds | int | `nil` | How often (in seconds) to perform the probe. | +| gc.readinessProbe.successThreshold | int | `nil` | Minimum consecutive successes for the probe before considering the pod as ready. | +| gc.readinessProbe.failureThreshold | int | `nil` | The number of consecutive failures for the probe before considering the pod as not ready. | +| gc.resources.limits | object | `{"memory":"1Gi"}` | Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-gc container. | +| gc.resources.requests | object | `{"cpu":"10m","memory":"128Mi"}` | Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-gc container. | +| gc.port | int | `8080` | Port on which to serve http for metrics and healthz endpoints. | +| gc.nodeSelector | object | `{}` | [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-gc pods. | +| gc.tolerations | list | `[]` | [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-gc pods. | +| gc.annotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-gc pods. | +| gc.labels | object | `{}` | [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-gc pods. | +| gc.deploymentAnnotations | object | `{}` | [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-gc Deployment. | +| gc.affinity | object | `{}` | [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-gc pods. | +| gc.podDisruptionBudget.enable | bool | `false` | Configure PodDisruptionBudget for the nfd-master Deployment. | +| gc.podDisruptionBudget.minAvailable | int | `1` | Minimum number (or percentage) of pods that must be available after the eviction. | +| gc.podDisruptionBudget.unhealthyPodEvictionPolicy | string | `"AlwaysAllow"` | Policy to evict unhealthy pods when a PodDisruptionBudget is defined. | +| gc.revisionHistoryLimit | int | `nil` | Specifies the number of old history for the Deployment to retain to allow rollback. | +| gc.networkPolicy.enabled | bool | `false` | Should a networkPolicy be deployed for the nfd-gc pods | +| gc.networkPolicy.egress | list | `[{"ports":[{"port":80,"protocol":"TCP"},{"port":443,"protocol":"TCP"},{"port":53,"protocol":"TCP"},{"port":53,"protocol":"UDP"},{"port":6443,"protocol":"TCP"}]}]` | [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-gc pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp | +| gc.networkPolicy.ingress | list | `[{"ports":[{"port":"http","protocol":"TCP"}]}]` | [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-gc pods. | + +### Prometheus + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| prometheus.enable | bool | `false` | Create PodMonitor object for enabling metrics collection by Prometheus Operator. | +| prometheus.scrapeInterval | string | `"10s"` | Interval at which metrics are scraped. | +| prometheus.labels | object | `{}` | Labels to add to the PodMonitor object. | diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md.gotmpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md.gotmpl new file mode 100644 index 00000000..90f7cce4 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/README.md.gotmpl @@ -0,0 +1,9 @@ +{{ template "chart.header" . }} +{{ template "chart.badgesSection" . }} +{{ template "chart.description" . }} +{{ template "chart.homepageLine" . }} +{{ template "chart.maintainersSection" . }} +{{ template "chart.sourcesSection" . }} +{{ template "chart.requirementsSection" . }} + +{{ template "common" . }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata-site.gotmpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata-site.gotmpl new file mode 100644 index 00000000..25444e18 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata-site.gotmpl @@ -0,0 +1,15 @@ +{{- define "metadata.oci_repo" -}} +{{ "{{ site.helm_oci_repo }}" }} +{{- end -}} + +{{- define "metadata.oci_tag" -}} +{{ "{{ site.helm_chart_version }}" }} +{{- end -}} + +{{- define "metadata.docs_version" -}} +{{ "{{ site.version }}" }} +{{- end -}} + +{{- define "metadata.nfd_release" -}} +{{ "{{ site.release }}" }} +{{- end -}} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata.gotmpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata.gotmpl new file mode 100644 index 00000000..7c9fa77d --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_metadata.gotmpl @@ -0,0 +1,11 @@ +{{- define "metadata.oci_repo" -}}oci://gcr.io/k8s-staging-nfd/charts/node-feature-discovery +{{- end -}} + +{{- define "metadata.oci_tag" -}}0.0.0-master +{{- end -}} + +{{- define "metadata.docs_version" -}}master +{{- end -}} + +{{- define "metadata.nfd_release" -}}master +{{- end -}} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_templates.gotmpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_templates.gotmpl new file mode 100644 index 00000000..989eeb9d --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/_templates.gotmpl @@ -0,0 +1,158 @@ +{{ define "note.autogenerated" -}} + +{{- end }} + +{{- define "common" }} + +## Installing the chart + +### OCI Helm repository + +The NFD project provides Helm charts in an OCI compliant repository. Install +NFD with the default configuration + +```bash +helm install nfd --namespace node-feature-discovery --create-namespace {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} +``` + +See the [configuration](#configuration) section below for instructions how to +alter the deployment parameters. + +### Legacy Helm repository + +For the time being, the NFD project still provides Helm charts in a legacy +(HTTP) Helm repository, too. + +> **NOTE:** This repository will be deprecated in the future. It is recommended +> for users to switch to use the OCI Helm repository. + +#### Stable version + +To install the latest stable version: + +```bash +helm repo add nfd https://kubernetes-sigs.github.io/node-feature-discovery/charts +helm repo update +helm install nfd nfd/node-feature-discovery --namespace node-feature-discovery --create-namespace +``` + +See the [configuration](#configuration) section below for instructions how to +alter the deployment parameters. + +### Latest development version + +To install the latest development version you need to clone the NFD Git +repository and install from there. + +```bash +git clone https://github.com/kubernetes-sigs/node-feature-discovery/ +cd node-feature-discovery/deployment/helm +helm install nfd ./node-feature-discovery/ --namespace node-feature-discovery --create-namespace +``` + +## Configuration + +You can override values from `values.yaml` and provide a file with custom values: + +```bash +helm install -f --namespace nfd --create-namespace nfd {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} +``` + +To specify each parameter separately you can provide them to helm install command: + +```bash +helm install --set nameOverride=NFDinstance --set master.replicaCount=2 --namespace nfd --create-namespace nfd {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} +``` + +## Upgrading the chart + +To upgrade the `node-feature-discovery` deployment to {{ template "metadata.nfd_release" . }} via Helm. + +### Rolling-update pace on large clusters + +The nfd-worker DaemonSet rolls out updates with `maxUnavailable: "10%"` by +default, so an upgrade completes in roughly ten waves regardless of cluster +size. nfd-worker is stateless and node labels persist while a worker pod +restarts, so tuning the pace only affects how quickly feature updates resume +on each node. Adjust it via `worker.updateStrategy.rollingUpdate.maxUnavailable` +(e.g. `1` restores the Kubernetes default of one node at a time; to use +`type: OnDelete`, also set `rollingUpdate: null` as Helm deep-merges maps). + +When driving upgrades through `helm upgrade --wait` or GitOps health checks +(Flux `HelmRelease`, Argo CD), size the timeout to cover the full rollout: +roughly `ceil(1 / maxUnavailable) × per-wave pod-ready time`. With the serial +Kubernetes default the rollout time grows linearly with node count and easily +exceeds a 5-minute timeout on large clusters. + +### From v0.7 and older + +Please see +the [uninstallation guide](https://kubernetes-sigs.github.io/node-feature-discovery/v0.7/get-started/deployment-and-usage.html#uninstallation). +And then follow the standard [installation instructions](#installing-the-chart). + +### From v0.8 - v0.11 + +Helm deployment of NFD was introduced in v0.8.0. + +```bash +export NFD_NS=node-feature-discovery +export HELM_INSTALL_NAME=nfd +# Uninstall the old NFD deployment +helm uninstall $HELM_INSTALL_NAME --namespace $NFD_NS +# Install the new NFD deployment +helm install $HELM_INSTALL_NAME {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} --namespace $NFD_NS --set master.enable=false +# Wait for NFD Worker to be ready +kubectl wait --timeout=-1s --for=condition=ready pod -l app.kubernetes.io/name=node-feature-discovery --namespace $NFD_NS +# Enable the NFD Master +helm upgrade $HELM_INSTALL_NAME {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} --namespace $NFD_NS --set master.enable=true +``` + +### From v0.12 - v0.13 + +In v0.12 the `NodeFeature` CRD was introduced as experimental. +The API was not enabled by default. + +```bash +export NFD_NS=node-feature-discovery +export HELM_INSTALL_NAME=nfd +# Install and upgrade CRD's +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/node-feature-discovery/{{ template "metadata.nfd_release" . }}/deployment/base/nfd-crds/nfd-api-crds.yaml +# Install the new NFD deployment +helm upgrade $HELM_INSTALL_NAME {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} --namespace $NFD_NS --set master.enable=false +# Wait for NFD Worker to be ready +kubectl wait --timeout=-1s --for=condition=ready pod -l app.kubernetes.io/name=node-feature-discovery --namespace $NFD_NS +# Enable the NFD Master +helm upgrade $HELM_INSTALL_NAME {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} --namespace $NFD_NS --set master.enable=true +``` + +### From v0.14+ + +As of version v0.14 the Helm chart is the primary deployment method for NFD, +and the CRD `NodeFeature` is enabled by default. + +```bash +export NFD_NS=node-feature-discovery +export HELM_INSTALL_NAME=nfd +# Install and upgrade CRD's +kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/node-feature-discovery/{{ template "metadata.nfd_release" . }}/deployment/base/nfd-crds/nfd-api-crds.yaml +# Install the new NFD deployment +helm upgrade $HELM_INSTALL_NAME {{ template "metadata.oci_repo" . }} --version {{ template "metadata.oci_tag" . }} --namespace $NFD_NS +``` + +## Uninstalling the chart + +To uninstall the `node-feature-discovery` deployment: + +```bash +helm uninstall nfd --namespace node-feature-discovery +``` + +The command removes all the Kubernetes components associated with the chart and +deletes the release. It also runs a post-delete hook that cleans up the nodes +of all labels, annotations, taints and extended resources that were created by +NFD. + +{{ template "chart.valuesSection" . }} +{{ end -}} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml new file mode 100644 index 00000000..69e306c3 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml @@ -0,0 +1,788 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodefeatures.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeature + listKind: NodeFeatureList + plural: nodefeatures + singular: nodefeature + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + NodeFeature resource holds the features discovered for one node in the + cluster. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the NodeFeature, containing features discovered + for a node. + properties: + features: + description: Features is the full "raw" features data that has been + discovered. + properties: + attributes: + additionalProperties: + description: AttributeFeatureSet is a set of features having + string value. + properties: + elements: + additionalProperties: + type: string + description: Individual features of the feature set. + type: object + required: + - elements + type: object + description: Attributes contains all the attribute-type features + of the node. + type: object + flags: + additionalProperties: + description: FlagFeatureSet is a set of simple features only + containing names without values. + properties: + elements: + additionalProperties: + description: |- + Nil is a dummy empty struct for protobuf compatibility. + NOTE: protobuf definitions have been removed but this is kept for API compatibility. + type: object + description: Individual features of the feature set. + type: object + required: + - elements + type: object + description: Flags contains all the flag-type features of the + node. + type: object + instances: + additionalProperties: + description: InstanceFeatureSet is a set of features each of + which is an instance having multiple attributes. + properties: + elements: + description: Individual features of the feature set. + items: + description: InstanceFeature represents one instance of + a complex features, e.g. a device. + properties: + attributes: + additionalProperties: + type: string + description: Attributes of the instance feature. + type: object + required: + - attributes + type: object + type: array + required: + - elements + type: object + description: Instances contains all the instance-type features + of the node. + type: object + type: object + labels: + additionalProperties: + type: string + description: Labels is the set of node labels that are requested to + be created. + type: object + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodefeaturegroups.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeatureGroup + listKind: NodeFeatureGroupList + plural: nodefeaturegroups + shortNames: + - nfg + singular: nodefeaturegroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeFeatureGroup resource holds Node pools by featureGroup + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the rules to be evaluated. + properties: + featureGroupRules: + description: List of rules to evaluate to determine nodes that belong + in this group. + items: + description: GroupRule defines a rule for nodegroup filtering. + properties: + matchAny: + description: MatchAny specifies a list of matchers one of which + must match. + items: + description: MatchAnyElem specifies one sub-matcher of MatchAny. + properties: + matchFeatures: + description: MatchFeatures specifies a set of matcher + terms all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature + set to match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + required: + - matchFeatures + type: object + type: array + matchFeatures: + description: MatchFeatures specifies a set of matcher terms + all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature set to + match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + name: + description: Name of the rule. + type: string + vars: + additionalProperties: + type: string + description: |- + Vars is the variables to store if the rule matches. Variables can be + referenced from other rules enabling more complex rule hierarchies. + type: object + varsTemplate: + description: |- + VarsTemplate specifies a template to expand for dynamically generating + multiple variables. Data (after template expansion) must be keys with an + optional value ([=]) separated by newlines. + type: string + required: + - name + type: object + type: array + required: + - featureGroupRules + type: object + status: + description: |- + Status of the NodeFeatureGroup after the most recent evaluation of the + specification. + properties: + nodes: + description: Nodes is a list of FeatureGroupNode in the cluster that + match the featureGroupRules + items: + properties: + name: + description: Name of the node. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodefeaturerules.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeatureRule + listKind: NodeFeatureRuleList + plural: nodefeaturerules + shortNames: + - nfr + singular: nodefeaturerule + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + NodeFeatureRule resource specifies a configuration for feature-based + customization of node objects, such as node labeling. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the rules to be evaluated. + properties: + rules: + description: Rules is a list of node customization rules. + items: + description: Rule defines a rule for node customization such as + labeling. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to create if the rule matches. + type: object + extendedResources: + additionalProperties: + type: string + description: ExtendedResources to create if the rule matches. + type: object + labels: + additionalProperties: + type: string + description: Labels to create if the rule matches. + type: object + labelsTemplate: + description: |- + LabelsTemplate specifies a template to expand for dynamically generating + multiple labels. Data (after template expansion) must be keys with an + optional value ([=]) separated by newlines. + type: string + matchAny: + description: MatchAny specifies a list of matchers one of which + must match. + items: + description: MatchAnyElem specifies one sub-matcher of MatchAny. + properties: + matchFeatures: + description: MatchFeatures specifies a set of matcher + terms all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature + set to match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + required: + - matchFeatures + type: object + type: array + matchFeatures: + description: MatchFeatures specifies a set of matcher terms + all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature set to + match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Ge + - Lt + - Le + - GtLt + - GeLe + - IsTrue + - IsFalse + type: string + type: + description: |- + Type defines the value type for specific operators. + The currently supported type is 'version' for Gt,Ge,Lt,Le,GtLt,GeLe operators. + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + name: + description: Name of the rule. + type: string + taints: + description: Taints to create if the rule matches. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array + vars: + additionalProperties: + type: string + description: |- + Vars is the variables to store if the rule matches. Variables do not + directly inflict any changes in the node object. However, they can be + referenced from other rules enabling more complex rule hierarchies, + without exposing intermediary output values as labels. + type: object + varsTemplate: + description: |- + VarsTemplate specifies a template to expand for dynamically generating + multiple variables. Data (after template expansion) must be keys with an + optional value ([=]) separated by newlines. + type: string + required: + - name + type: object + type: array + required: + - rules + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/_helpers.tpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/_helpers.tpl new file mode 100644 index 00000000..b540e3bb --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/_helpers.tpl @@ -0,0 +1,126 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "node-feature-discovery.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "node-feature-discovery.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "node-feature-discovery.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "node-feature-discovery.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "node-feature-discovery.labels" -}} +helm.sh/chart: {{ include "node-feature-discovery.chart" . }} +{{ include "node-feature-discovery.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "node-feature-discovery.selectorLabels" -}} +app.kubernetes.io/name: {{ include "node-feature-discovery.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account which the nfd master will use +*/}} +{{- define "node-feature-discovery.master.serviceAccountName" -}} +{{- if .Values.master.serviceAccount.create -}} + {{ default (include "node-feature-discovery.fullname" .) .Values.master.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.master.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account which the nfd worker will use +*/}} +{{- define "node-feature-discovery.worker.serviceAccountName" -}} +{{- if .Values.worker.serviceAccount.create -}} + {{ default (printf "%s-worker" (include "node-feature-discovery.fullname" .)) .Values.worker.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.worker.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account which topologyUpdater will use +*/}} +{{- define "node-feature-discovery.topologyUpdater.serviceAccountName" -}} +{{- if .Values.topologyUpdater.serviceAccount.create -}} + {{ default (printf "%s-topology-updater" (include "node-feature-discovery.fullname" .)) .Values.topologyUpdater.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.topologyUpdater.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account which nfd-gc will use +*/}} +{{- define "node-feature-discovery.gc.serviceAccountName" -}} +{{- if .Values.gc.serviceAccount.create -}} + {{ default (printf "%s-gc" (include "node-feature-discovery.fullname" .)) .Values.gc.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.gc.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +imagePullSecrets helper - uses local values or falls back to global values +*/}} +{{- define "node-feature-discovery.imagePullSecrets" -}} +{{- $imagePullSecrets := list -}} +{{- if .Values.imagePullSecrets -}} + {{- range .Values.imagePullSecrets -}} + {{- $imagePullSecrets = append $imagePullSecrets . -}} + {{- end -}} +{{- else if and .Values.global .Values.global.imagePullSecrets -}} + {{- range .Values.global.imagePullSecrets -}} + {{- $imagePullSecrets = append $imagePullSecrets . -}} + {{- end -}} +{{- end -}} +{{- if $imagePullSecrets -}} +{{- $imagePullSecrets | toJson }} +{{- end -}} +{{- end -}} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml new file mode 100644 index 00000000..437864ad --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml @@ -0,0 +1,162 @@ +{{- if and .Values.master.enable .Values.master.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - watch + - list +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + - nodefeaturerules + - nodefeaturegroups + verbs: + - get + - list + - watch +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeaturegroups/status + verbs: + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - "nfd-master.nfd.kubernetes.io" + verbs: + - get + - update +{{- end }} + +{{- if and .Values.worker.enable .Values.worker.rbac.create }} +{{- if has "node" .Values.worker.ownerRefs }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +{{- end }} +{{- end }} + +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - nodes/configz + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - create + - patch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +{{- end }} + +{{- if and .Values.gc.enable .Values.gc.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - delete + - list +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..16902cef --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml @@ -0,0 +1,72 @@ +{{- if and .Values.master.enable .Values.master.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.master.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + +{{- if and .Values.worker.enable .Values.worker.rbac.create }} +{{- if has "node" .Values.worker.ownerRefs }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-worker +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} +{{- end }} + +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + +{{- if and .Values.gc.enable .Values.gc.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-gc +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.gc.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-networkpolicy.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-networkpolicy.yaml new file mode 100644 index 00000000..3081f3ff --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-networkpolicy.yaml @@ -0,0 +1,26 @@ +{{- if .Values.master.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "node-feature-discovery.fullname" . }}-master-network-policy + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: master +spec: + egress: + {{- with .Values.master.networkPolicy.egress }} + {{- toYaml . | nindent 4 }} + {{- end }} + ingress: + {{- with .Values.master.networkPolicy.ingress }} + {{- toYaml . | nindent 4 }} + {{- end }} + podSelector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: master + policyTypes: + - Egress + - Ingress +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-pdb.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-pdb.yaml new file mode 100644 index 00000000..532b0e6d --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master-pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.master.enable }} +{{- if .Values.master.podDisruptionBudget.enable -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: master +{{- toYaml (omit .Values.master.podDisruptionBudget "enable") | nindent 2 }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master.yaml new file mode 100644 index 00000000..8fa8e2f1 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/master.yaml @@ -0,0 +1,181 @@ +{{- if .Values.master.enable }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: master + {{- with .Values.master.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.master.replicaCount }} + revisionHistoryLimit: {{ .Values.master.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: master + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + {{- with .Values.master.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + role: master + annotations: + {{- if .Values.checksumValuesOnly }} + checksum/config: {{ .Values.master.config | toYaml | sha256sum }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/nfd-master-conf.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.master.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + dnsPolicy: {{ .Values.master.dnsPolicy }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + serviceAccountName: {{ include "node-feature-discovery.master.serviceAccountName" . }} + enableServiceLinks: false + securityContext: + {{- toYaml .Values.master.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.master.hostNetwork }} + {{- if kindIs "bool" .Values.master.hostUsers }} + hostUsers: {{ .Values.master.hostUsers }} + {{- end }} + {{- with .Values.master.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: master + securityContext: + {{- toYaml .Values.master.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + startupProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.master.startupProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.master.startupProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.master.startupProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.master.startupProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.master.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.master.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.master.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.master.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.master.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + ports: + - containerPort: {{ .Values.master.port | default "8080" }} + name: http + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.master.extraEnvs }} + {{- toYaml . | nindent 10 }} + {{- end}} + command: + - "nfd-master" + resources: + {{- toYaml .Values.master.resources | nindent 12 }} + args: + {{- if .Values.master.instance | empty | not }} + - "-instance={{ .Values.master.instance }}" + {{- end }} + {{- if gt (int .Values.master.replicaCount) 1 }} + - "-enable-leader-election" + {{- end }} + {{- if .Values.master.extraLabelNs | empty | not }} + - "-extra-label-ns={{- join "," .Values.master.extraLabelNs }}" + {{- end }} + {{- if .Values.master.denyLabelNs | empty | not }} + - "-deny-label-ns={{- join "," .Values.master.denyLabelNs }}" + {{- end }} + {{- if .Values.master.enableTaints }} + - "-enable-taints" + {{- end }} + {{- if .Values.master.resyncPeriod }} + - "-resync-period={{ .Values.master.resyncPeriod }}" + {{- end }} + {{- if .Values.master.nfdApiParallelism | empty | not }} + - "-nfd-api-parallelism={{ .Values.master.nfdApiParallelism }}" + {{- end }} + # Go over featureGates and add the feature-gate flag + {{- range $key, $value := .Values.featureGates }} + - "-feature-gates={{ $key }}={{ $value }}" + {{- end }} + - "-port={{ .Values.master.port | default "8080" }}" + {{- with .Values.master.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: nfd-master-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-master-conf + items: + - key: nfd-master.conf + path: nfd-master.conf + {{- with .Values.master.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-networkpolicy.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-networkpolicy.yaml new file mode 100644 index 00000000..1772de84 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-networkpolicy.yaml @@ -0,0 +1,26 @@ +{{- if .Values.gc.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "node-feature-discovery.fullname" . }}-gc-network-policy + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: gc +spec: + egress: + {{- with .Values.gc.networkPolicy.egress }} + {{- toYaml . | nindent 4 }} + {{- end }} + ingress: + {{- with .Values.gc.networkPolicy.ingress }} + {{- toYaml . | nindent 4 }} + {{- end }} + podSelector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: gc + policyTypes: + - Egress + - Ingress +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml new file mode 100644 index 00000000..373ed92c --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc-pdb.yaml @@ -0,0 +1,17 @@ +{{- if .Values.gc.enable }} +{{- if .Values.gc.podDisruptionBudget.enable -}} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: gc +{{- toYaml (omit .Values.gc.podDisruptionBudget "enable") | nindent 2 }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml new file mode 100644 index 00000000..8b0b7ca6 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml @@ -0,0 +1,130 @@ +{{- if and .Values.gc.enable -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: gc + {{- with .Values.gc.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.gc.replicaCount | default 1 }} + revisionHistoryLimit: {{ .Values.gc.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: gc + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + {{- with .Values.gc.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + role: gc + {{- with .Values.gc.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "node-feature-discovery.gc.serviceAccountName" . }} + dnsPolicy: {{ .Values.gc.dnsPolicy }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + securityContext: + {{- toYaml .Values.gc.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.gc.hostNetwork }} + {{- if kindIs "bool" .Values.gc.hostUsers }} + hostUsers: {{ .Values.gc.hostUsers }} + {{- end }} + {{- with .Values.gc.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: gc + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + livenessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.gc.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.gc.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.gc.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.gc.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.gc.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.gc.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.gc.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + command: + - "nfd-gc" + args: + {{- if .Values.gc.interval | empty | not }} + - "-gc-interval={{ .Values.gc.interval }}" + {{- end }} + {{- with .Values.gc.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} + resources: + {{- toYaml .Values.gc.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + ports: + - name: http + containerPort: {{ .Values.gc.port | default "8080"}} + + {{- with .Values.gc.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gc.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gc.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml new file mode 100644 index 00000000..9c6e01cd --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml @@ -0,0 +1,12 @@ +{{- if .Values.master.enable }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-master.conf: |- + {{- .Values.master.config | toYaml | nindent 4 }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml new file mode 100644 index 00000000..8d03aa2d --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml @@ -0,0 +1,12 @@ +{{- if .Values.topologyUpdater.enable -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-topology-updater.conf: |- + {{- .Values.topologyUpdater.config | toYaml | nindent 4 }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml new file mode 100644 index 00000000..a2299dea --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml @@ -0,0 +1,12 @@ +{{- if .Values.worker.enable }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-worker.conf: |- + {{- .Values.worker.config | toYaml | nindent 4 }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml new file mode 100644 index 00000000..fb5518b6 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml @@ -0,0 +1,102 @@ +{{- if .Values.postDeleteCleanup }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-prune +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.fullname" . }}-prune + namespace: {{ include "node-feature-discovery.namespace" . }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + {{- include "node-feature-discovery.labels" . | nindent 8 }} + role: prune + spec: + serviceAccountName: {{ include "node-feature-discovery.fullname" . }}-prune + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + containers: + - name: nfd-master + securityContext: + {{- toYaml .Values.master.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "nfd-master" + args: + - "-prune" + {{- if .Values.master.instance | empty | not }} + - "-instance={{ .Values.master.instance }}" + {{- end }} + {{- with .Values.master.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + restartPolicy: Never + {{- with .Values.master.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/prometheus.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/prometheus.yaml new file mode 100644 index 00000000..bed37f4c --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/prometheus.yaml @@ -0,0 +1,26 @@ +{{- if .Values.prometheus.enable }} +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "node-feature-discovery.fullname" . }} + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 4 }} + {{- with .Values.prometheus.labels }} + {{ toYaml . | nindent 4 }} + {{- end }} +spec: + podMetricsEndpoints: + - honorLabels: true + interval: {{ .Values.prometheus.scrapeInterval }} + path: /metrics + port: http + scheme: http + namespaceSelector: + matchNames: + - {{ include "node-feature-discovery.namespace" . }} + selector: + matchExpressions: + - {key: app.kubernetes.io/instance, operator: In, values: ["{{ .Release.Name }}"]} + - {key: app.kubernetes.io/name, operator: In, values: ["{{ include "node-feature-discovery.name" . }}"]} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/role.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/role.yaml new file mode 100644 index 00000000..ed12ef2c --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/role.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.worker.enable .Values.worker.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update + - delete +- apiGroups: + - "" + resources: + - pods + verbs: + - get +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml new file mode 100644 index 00000000..a640d5f8 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.worker.enable .Values.worker.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "node-feature-discovery.fullname" . }}-worker +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml new file mode 100644 index 00000000..47c75a7e --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.master.enable .Values.master.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.master.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.master.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.topologyUpdater.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if and .Values.gc.enable .Values.gc.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.gc.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.gc.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if and .Values.worker.enable .Values.worker.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.worker.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml new file mode 100644 index 00000000..b6b91968 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml @@ -0,0 +1,278 @@ +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.createCRDs -}} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 + controller-gen.kubebuilder.io/version: v0.11.2 + creationTimestamp: null + name: noderesourcetopologies.topology.node.k8s.io +spec: + group: topology.node.k8s.io + names: + kind: NodeResourceTopology + listKind: NodeResourceTopologyList + plural: noderesourcetopologies + shortNames: + - node-res-topo + singular: noderesourcetopology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - topologyPolicies + - zones + type: object + served: true + storage: false + - name: v1alpha2 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + description: 'DEPRECATED (to be removed in v1beta1): use top level attributes + if needed' + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - zones + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-networkpolicy.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-networkpolicy.yaml new file mode 100644 index 00000000..2cd6cc25 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater-networkpolicy.yaml @@ -0,0 +1,26 @@ +{{- if .Values.topologyUpdater.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "node-feature-discovery.fullname" . }}-topology-updater-network-policy + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: topology-updater +spec: + egress: + {{- with .Values.topologyUpdater.networkPolicy.egress }} + {{- toYaml . | nindent 4 }} + {{- end }} + ingress: + {{- with .Values.topologyUpdater.networkPolicy.ingress }} + {{- toYaml . | nindent 4 }} + {{- end }} + podSelector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: topology-updater + policyTypes: + - Egress + - Ingress +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml new file mode 100644 index 00000000..0d25a503 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml @@ -0,0 +1,198 @@ +{{- if .Values.topologyUpdater.enable -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: topology-updater + {{- with .Values.topologyUpdater.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + revisionHistoryLimit: {{ .Values.topologyUpdater.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: topology-updater + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + {{- with .Values.topologyUpdater.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + role: topology-updater + annotations: + {{- if .Values.checksumValuesOnly }} + checksum/config: {{ .Values.topologyUpdater.config | toYaml | sha256sum }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/nfd-topologyupdater-conf.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.topologyUpdater.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} + dnsPolicy: {{ .Values.topologyUpdater.dnsPolicy }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + securityContext: + {{- toYaml .Values.topologyUpdater.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.topologyUpdater.hostNetwork }} + {{- if kindIs "bool" .Values.topologyUpdater.hostUsers }} + hostUsers: {{ .Values.topologyUpdater.hostUsers }} + {{- end }} + {{- with .Values.topologyUpdater.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: topology-updater + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + livenessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.topologyUpdater.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.topologyUpdater.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- with .Values.topologyUpdater.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + command: + - "nfd-topology-updater" + args: + - "-podresources-socket=/host-var/lib/kubelet-podresources/kubelet.sock" + {{- if .Values.topologyUpdater.updateInterval | empty | not }} + - "-sleep-interval={{ .Values.topologyUpdater.updateInterval }}" + {{- else }} + - "-sleep-interval=3s" + {{- end }} + {{- if .Values.topologyUpdater.watchNamespace | empty | not }} + - "-watch-namespace={{ .Values.topologyUpdater.watchNamespace }}" + {{- else }} + - "-watch-namespace=*" + {{- end }} + {{- if not .Values.topologyUpdater.podSetFingerprint }} + - "-pods-fingerprint=false" + {{- end }} + {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} + - "-kubelet-config-uri=file:///host-var/kubelet-config" + {{- end }} + {{- if .Values.topologyUpdater.kubeletStateDir | empty }} + # Disable kubelet state tracking by giving an empty path + - "-kubelet-state-dir=" + {{- end }} + - "-port={{ .Values.topologyUpdater.port | default "8080"}}" + {{- with .Values.topologyUpdater.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} + ports: + - containerPort: {{ .Values.topologyUpdater.port | default "8080"}} + name: http + volumeMounts: + {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} + - name: kubelet-config + mountPath: /host-var/kubelet-config + {{- end }} + - name: kubelet-podresources-sock + mountPath: /host-var/lib/kubelet-podresources/kubelet.sock + - name: host-sys + mountPath: /host-sys + {{- if .Values.topologyUpdater.kubeletStateDir | empty | not }} + - name: kubelet-state-files + mountPath: /host-var/lib/kubelet + readOnly: true + {{- end }} + - name: nfd-topology-updater-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + + resources: + {{- toYaml .Values.topologyUpdater.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.topologyUpdater.securityContext | nindent 12 }} + volumes: + - name: host-sys + hostPath: + path: "/sys" + {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} + - name: kubelet-config + hostPath: + path: {{ .Values.topologyUpdater.kubeletConfigPath }} + {{- end }} + - name: kubelet-podresources-sock + hostPath: + {{- if .Values.topologyUpdater.kubeletPodResourcesSockPath | empty | not }} + path: {{ .Values.topologyUpdater.kubeletPodResourcesSockPath }} + {{- else }} + path: /var/lib/kubelet/pod-resources/kubelet.sock + {{- end }} + {{- if .Values.topologyUpdater.kubeletStateDir | empty | not }} + - name: kubelet-state-files + hostPath: + path: {{ .Values.topologyUpdater.kubeletStateDir }} + {{- end }} + - name: nfd-topology-updater-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater-conf + items: + - key: nfd-topology-updater.conf + path: nfd-topology-updater.conf + + {{- with .Values.topologyUpdater.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologyUpdater.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologyUpdater.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker-networkpolicy.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker-networkpolicy.yaml new file mode 100644 index 00000000..85e1ffcd --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker-networkpolicy.yaml @@ -0,0 +1,26 @@ +{{- if .Values.worker.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ template "node-feature-discovery.fullname" . }}-worker-network-policy + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: worker +spec: + egress: + {{- with .Values.worker.networkPolicy.egress }} + {{- toYaml . | nindent 4 }} + {{- end }} + ingress: + {{- with .Values.worker.networkPolicy.ingress }} + {{- toYaml . | nindent 4 }} + {{- end }} + podSelector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: worker + policyTypes: + - Egress + - Ingress +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker.yaml new file mode 100644 index 00000000..ab321294 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/templates/worker.yaml @@ -0,0 +1,210 @@ +{{- if .Values.worker.enable }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: worker + {{- with .Values.worker.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + revisionHistoryLimit: {{ .Values.worker.revisionHistoryLimit }} + {{- with .Values.worker.updateStrategy }} + updateStrategy: + {{- toYaml . | nindent 4 }} + {{- end}} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: worker + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + {{- with .Values.worker.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + role: worker + annotations: + {{- if .Values.checksumValuesOnly }} + checksum/config: {{ .Values.worker.config | toYaml | sha256sum }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/nfd-worker-conf.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.worker.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + dnsPolicy: {{ .Values.worker.dnsPolicy }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + imagePullSecrets: {{ include "node-feature-discovery.imagePullSecrets" . }} + serviceAccountName: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.worker.hostNetwork }} + {{- if kindIs "bool" .Values.worker.hostUsers }} + hostUsers: {{ .Values.worker.hostUsers }} + {{- end }} + {{- with .Values.worker.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.worker.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.worker.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.worker.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.worker.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + httpGet: + path: /healthz + port: http + {{- with .Values.worker.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + {{- with .Values.worker.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + command: + - "nfd-worker" + args: + # Go over featureGate and add the feature-gate flag + {{- range $key, $value := .Values.featureGates }} + - "-feature-gates={{ $key }}={{ $value }}" + {{- end }} + - "-owner-refs={{ join "," .Values.worker.ownerRefs }}" + - "-port={{ .Values.worker.port | default "8080"}}" + {{- with .Values.worker.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.worker.port | default "8080"}} + name: http + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + readOnly: true + - name: host-usr-lib + mountPath: "/host-usr/lib" + readOnly: true + - name: host-lib + mountPath: "/host-lib" + readOnly: true + - name: host-proc-swaps + mountPath: "/host-proc/swaps" + readOnly: true + {{- if .Values.worker.mountUsrSrc }} + - name: host-usr-src + mountPath: "/host-usr/src" + readOnly: true + {{- end }} + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + readOnly: true + - name: nfd-worker-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: host-boot + hostPath: + path: "/boot" + - name: host-os-release + hostPath: + path: "/etc/os-release" + - name: host-sys + hostPath: + path: "/sys" + - name: host-usr-lib + hostPath: + path: "/usr/lib" + - name: host-lib + hostPath: + path: "/lib" + - name: host-proc-swaps + hostPath: + path: "/proc/swaps" + {{- if .Values.worker.mountUsrSrc }} + - name: host-usr-src + hostPath: + path: "/usr/src" + {{- end }} + - name: features-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/features.d/" + - name: nfd-worker-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-worker-conf + items: + - key: nfd-worker.conf + path: nfd-worker.conf + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.schema.json b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.schema.json new file mode 100644 index 00000000..83591103 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.schema.json @@ -0,0 +1,4622 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "checksumValuesOnly": { + "description": "Only checksum config values (not the full rendered ConfigMap template) for the rollout annotation. When enabled, pods will only restart when config values actually change, not on unrelated template changes.", + "type": "boolean" + }, + "featureGates": { + "description": "[Feature gates](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/feature-gates) to enable/disable specific features.", + "type": "object", + "patternProperties": { + "^[a-zA-Z]$": { + "type": "string" + } + } + }, + "fullnameOverride": { + "description": "Override a default fully qualified app name", + "type": "string" + }, + "gc": { + "type": "object", + "properties": { + "affinity": { + "description": "[Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Affinity", + "type": "object" + }, + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "deploymentAnnotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-gc Deployment.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "dnsPolicy": { + "description": "NFD gc pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy).", + "type": "string" + }, + "enable": { + "description": "Specifies whether nfd-gc should be deployed.", + "type": "boolean" + }, + "extraArgs": { + "description": "Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/gc-commandline-reference) to pass to nfd-gc.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/args", + "type": "array" + }, + "extraEnvs": { + "description": "Additional environment variables to set in the nfd-gc container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/env", + "type": "array" + }, + "hostNetwork": { + "description": "Run the container in the host's network namespace.", + "type": "boolean" + }, + "hostUsers": { + "description": "Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true.", + "type": [ + "boolean", + "null" + ] + }, + "initContainers": { + "description": "[Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers", + "type": "array" + }, + "interval": { + "description": "Time between periodic garbage collector runs.", + "type": "string" + }, + "labels": { + "description": "[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels", + "type": "object" + }, + "livenessProbe": { + "description": "Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "egress": { + "description": "[Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-gc pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule", + "type": "object" + } + }, + "enabled": { + "description": "Should a networkPolicy be deployed for the nfd-gc pods", + "type": "boolean" + }, + "ingress": { + "description": "[Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-gc pods.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule", + "type": "object" + } + } + } + }, + "nodeSelector": { + "description": "[Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector", + "type": "object" + }, + "podDisruptionBudget": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.policy.v1.PodDisruptionBudgetSpec", + "type": "object", + "properties": { + "enable": { + "description": "Configure PodDisruptionBudget for the nfd-master Deployment.", + "type": "boolean" + } + } + }, + "podSecurityContext": { + "description": "[Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSecurityContext", + "type": "object" + }, + "port": { + "description": "Port on which to serve http for metrics and healthz endpoints.", + "type": "integer", + "maximum": 65535, + "minimum": 1 + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "description": "Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-gc.", + "type": "boolean" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "The number of consecutive failures for the probe before considering the pod as not ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "successThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "replicaCount": { + "description": "The number of desired replicas for the nfd-gc Deployment.", + "type": "integer" + }, + "resources": { + "type": "object" + }, + "revisionHistoryLimit": { + "description": "Specifies the number of old history for the Deployment to retain to allow rollback.", + "type": [ + "integer", + "null" + ] + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "create": { + "description": "Specifies whether a service account should be created.", + "type": "boolean" + }, + "name": { + "description": "Name of the service account to use. If not set and create is true, a name is generated using the fullname template.", + "type": [ + "string", + "null" + ] + } + } + }, + "tolerations": { + "description": "[Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-gc pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations", + "type": "array" + } + } + }, + "global": { + "type": "object", + "properties": { + "imagePullSecrets": { + "description": "If `imagePullSecrets` is specified, it takes precedence over `global.imagePullSecrets`. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod).", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/imagePullSecrets", + "type": "array" + } + } + }, + "image": { + "type": "object", + "properties": { + "pullPolicy": { + "description": "Image pull policy", + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + }, + "repository": { + "description": "NFD image repository", + "type": "string" + }, + "tag": { + "description": "NFD image tag. If not specified Chart.AppVersion will be used.", + "type": [ + "string", + "null" + ] + } + } + }, + "imagePullSecrets": { + "description": "Image pull secrets. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod).", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/imagePullSecrets", + "type": "array" + }, + "master": { + "type": "object", + "properties": { + "affinity": { + "description": "[Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Affinity", + "type": "object" + }, + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "config": { + "description": "NFD master [configuration](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/master-configuration-reference).", + "type": [ + "object", + "null" + ] + }, + "denyLabelNs": { + "description": "Label namespaces to deny. Labels with these prefixes will not be published to the nodes.", + "type": "array", + "items": { + "type": "string" + } + }, + "deploymentAnnotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-master Deployment.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "dnsPolicy": { + "description": "NFD master pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy).", + "type": "string" + }, + "enable": { + "description": "Specifies whether nfd-master should be deployed", + "type": "boolean" + }, + "enableTaints": { + "description": "Enable node tainting.", + "type": "boolean" + }, + "extraArgs": { + "description": "Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/master-commandline-reference) to pass to nfd-master.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/args", + "type": "array" + }, + "extraEnvs": { + "description": "Additional environment variables to set in the nfd-master container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/env", + "type": "array" + }, + "extraLabelNs": { + "description": "Additional label namespaces to publish. Labels with these prefixes are allowed even if otherwise denied by `master.denyLabelNs`.", + "type": "array", + "items": { + "type": "string" + } + }, + "hostNetwork": { + "description": "Run the container in the host's network namespace.", + "type": "boolean" + }, + "hostUsers": { + "description": "Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true.", + "type": [ + "boolean", + "null" + ] + }, + "initContainers": { + "description": "[Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers", + "type": "array" + }, + "instance": { + "description": "Instance name. Used to separate annotation namespaces for multiple parallel deployments.", + "type": [ + "string", + "null" + ] + }, + "labels": { + "description": "[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels", + "type": "object" + }, + "livenessProbe": { + "description": "Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "egress": { + "description": "[Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-master pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule", + "type": "object" + } + }, + "enabled": { + "description": "Should a networkPolicy be deployed for the nfd-master pods", + "type": "boolean" + }, + "ingress": { + "description": "[Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-master pods.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule", + "type": "object" + } + } + } + }, + "nfdApiParallelism": { + "description": "The maximum number of concurrent node updates.", + "type": [ + "integer", + "null" + ], + "maximum": 100, + "minimum": 1 + }, + "nodeSelector": { + "description": "[Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector", + "type": "object" + }, + "podDisruptionBudget": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.policy.v1.PodDisruptionBudgetSpec", + "type": "object", + "properties": { + "enable": { + "description": "Configure PodDisruptionBudget for the nfd-master Deployment.", + "type": "boolean" + } + } + }, + "podSecurityContext": { + "description": "[Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSecurityContext", + "type": "object" + }, + "port": { + "description": "Port on which to serve http for metrics and healthz endpoints.", + "type": "integer", + "maximum": 65535, + "minimum": 1 + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "description": "Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-master.", + "type": "boolean" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "The number of consecutive failures for the probe before considering the pod as not ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "successThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "replicaCount": { + "description": "Number of the desired replicas for the nfd-master Deployment.", + "type": "integer" + }, + "resources": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceRequirements", + "type": "object" + }, + "resyncPeriod": { + "description": "NFD API controller resync period. Time duration string (e.g. \"5m\", \"1h\", \"2h45m\").", + "type": [ + "string", + "null" + ] + }, + "revisionHistoryLimit": { + "description": "Specifies the number of old ReplicaSets for the Deployment to retain. [revisionHistoryLimit](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#revision-history-limit)", + "type": [ + "integer", + "null" + ] + }, + "securityContext": { + "description": "[SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-master container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecurityContext", + "type": "object" + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "create": { + "description": "Specifies whether a service account should be created.", + "type": "boolean" + }, + "name": { + "description": "The name of the service account to use. If not set and create is true, a name is generated using the fullname template", + "type": [ + "string", + "null" + ] + } + } + }, + "startupProbe": { + "description": "Startup probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-startup-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "The number of consecutive failures for the probe before considering the pod as not ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "tolerations": { + "description": "[Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-master pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations", + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "nameOverride": { + "description": "Override the name of the chart", + "type": "string" + }, + "namespaceOverride": { + "description": "Override the namespace to install the chart into. By default, the namespace is determined by Helm.", + "type": "string" + }, + "postDeleteCleanup": { + "description": "Enable/disable the Helm post-delete hook.", + "type": "boolean" + }, + "priorityClassName": { + "description": "The name of the PriorityClass to be used for the NFD pods.", + "type": "string" + }, + "prometheus": { + "type": "object", + "properties": { + "enable": { + "description": "Create PodMonitor object for enabling metrics collection by Prometheus Operator.", + "type": "boolean" + }, + "labels": { + "description": "Labels to add to the PodMonitor object.", + "type": "object" + }, + "scrapeInterval": { + "description": "Interval at which metrics are scraped.", + "type": "string" + } + } + }, + "topologyUpdater": { + "type": "object", + "properties": { + "affinity": { + "description": "[Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Affinity", + "type": "object" + }, + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "config": { + "description": "Configuration for the topology updater. See the [configuration reference](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/topology-updater-configuration-reference) for details.", + "type": [ + "object", + "null" + ] + }, + "createCRDs": { + "description": "Create the NodeResourceTopology CRD. This MUST be set to true when 'enable' is true, unless the CRD is installed separately (e.g., by another Helm release or external tool). If the CRD is missing, the topology-updater pods will fail with \"NodeResourceTopology CRD is not installed\" error.", + "type": "boolean" + }, + "daemonsetAnnotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-topology-updater DaemonSet.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "dnsPolicy": { + "description": "NFD topology updater pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy).", + "type": "string" + }, + "enable": { + "description": "Specifies whether nfd-topology-updater should be deployed.", + "type": "boolean" + }, + "extraArgs": { + "description": "Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/topology-updater-commandline-reference) to pass to nfd-topology-updater.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/args", + "type": "array" + }, + "extraEnvs": { + "description": "Additional environment variables to set in the nfd-topology-updater container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/env", + "type": "array" + }, + "hostNetwork": { + "description": "Run the container in the host's network namespace.", + "type": "boolean" + }, + "hostUsers": { + "description": "Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true.", + "type": [ + "boolean", + "null" + ] + }, + "initContainers": { + "description": "[Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers", + "type": "array" + }, + "kubeletConfigPath": { + "description": "Host path for the kubelet config file.", + "type": [ + "string", + "null" + ] + }, + "kubeletPodResourcesSockPath": { + "description": "Host path for the kubelet socket for podresources endpoint.", + "type": [ + "string", + "null" + ] + }, + "kubeletStateDir": { + "description": "The kubelet state directory path for watching state and checkpoint files. Empty value disables kubelet state tracking.", + "type": "string" + }, + "labels": { + "description": "[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels", + "type": "object" + }, + "livenessProbe": { + "description": "Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "networkPolicy": { + "type": "object", + "properties": { + "egress": { + "description": "[Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-topology pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule", + "type": "object" + } + }, + "enabled": { + "description": "Should a networkPolicy be deployed for the nfd-topology pods", + "type": "boolean" + }, + "ingress": { + "description": "[Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-topology pods.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule", + "type": "object" + } + } + } + }, + "nodeSelector": { + "description": "[Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector", + "type": "object" + }, + "podSecurityContext": { + "description": "[Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSecurityContext", + "type": "object" + }, + "podSetFingerprint": { + "description": "Enables compute and report of pod fingerprint in NRT objects.", + "type": "boolean" + }, + "port": { + "description": "Port on which to serve http for metrics and healthz endpoints.", + "type": "integer", + "maximum": 65535, + "minimum": 1 + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "description": "Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-topology-updater.", + "type": "boolean" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "The number of consecutive failures for the probe before considering the pod as not ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "successThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "resources": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceRequirements", + "type": "object" + }, + "revisionHistoryLimit": { + "description": "Specifies the number of old history for the DaemonSet to retain to allow rollback.", + "type": [ + "integer", + "null" + ] + }, + "securityContext": { + "description": "[SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-topology-updater container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecurityContext", + "type": "object" + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "create": { + "description": "Specifies whether a service account should be created.", + "type": "boolean" + }, + "name": { + "description": "Name or the service account to use. If not set and create is true, a name is generated using the fullname template.", + "type": [ + "string", + "null" + ] + } + } + }, + "tolerations": { + "description": "[Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-topology-updater pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations", + "type": "array" + }, + "updateInterval": { + "description": "Time to sleep between CR updates. Non-positive value implies no CR update.", + "type": "string" + }, + "watchNamespace": { + "description": "Namespace to watch pods, `*` for all namespaces.", + "type": "string" + } + } + }, + "worker": { + "type": "object", + "properties": { + "affinity": { + "description": "[Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Affinity", + "type": "object" + }, + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "config": { + "description": "NFD worker [configuration](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/worker-configuration-reference).", + "type": [ + "object", + "null" + ] + }, + "daemonsetAnnotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-worker DaemonSet.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "dnsPolicy": { + "description": "NFD worker pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy).", + "type": "string" + }, + "enable": { + "description": "Specifies whether nfd-worker should be deployed", + "type": "boolean" + }, + "extraArgs": { + "description": "Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/worker-commandline-reference) to pass to nfd-worker.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/args", + "type": "array" + }, + "extraEnvs": { + "description": "Additional environment variables to set in the nfd-worker container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container/properties/env", + "type": "array" + }, + "hostNetwork": { + "description": "Run the container in the host's network namespace.", + "type": "boolean" + }, + "hostUsers": { + "description": "Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true.", + "type": [ + "boolean", + "null" + ] + }, + "initContainers": { + "description": "[Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers", + "type": "array" + }, + "labels": { + "description": "[Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels", + "type": "object" + }, + "livenessProbe": { + "description": "Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "mountUsrSrc": { + "description": "Mount host path /user/src inside the container. Does not work on systems without /usr/src AND a read-only /usr.", + "type": "boolean" + }, + "networkPolicy": { + "type": "object", + "properties": { + "egress": { + "description": "[Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-worker pods. The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule", + "type": "object" + } + }, + "enabled": { + "description": "Should a networkPolicy be deployed for the nfd-worker pods", + "type": "boolean" + }, + "ingress": { + "description": "[Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-worker pods.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule", + "type": "object" + } + } + } + }, + "nodeSelector": { + "description": "[Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector", + "type": "object" + }, + "ownerRefs": { + "description": "Objects used as owner references for NodeFeature objects. Valid values are `node`, `pod`, and `ds`. This value is passed through `-owner-refs` and takes precedence over `worker.config.core.ownerRefs`.", + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "enum": [ + "node", + "pod", + "ds" + ] + } + }, + "podSecurityContext": { + "description": "[Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSecurityContext", + "type": "object" + }, + "port": { + "description": "Port on which to serve http for metrics and healthz endpoints.", + "type": "integer", + "maximum": 65535, + "minimum": 1 + }, + "priorityClassName": { + "description": "The name of the PriorityClass to be used for the nfd-worker pods.", + "type": [ + "string", + "null" + ] + }, + "rbac": { + "type": "object", + "properties": { + "create": { + "description": "Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-worker.", + "type": "boolean" + } + } + }, + "readinessProbe": { + "description": "Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes).", + "type": "object", + "properties": { + "failureThreshold": { + "description": "The number of consecutive failures for the probe before considering the pod as not ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "initialDelaySeconds": { + "description": "The number of seconds after the container has started before probe is initiated.", + "type": [ + "integer", + "null" + ] + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "successThreshold": { + "description": "Minimum consecutive successes for the probe before considering the pod as ready.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + }, + "timeoutSeconds": { + "description": "The number of seconds after which the probe times out.", + "type": [ + "integer", + "null" + ], + "minimum": 1 + } + } + }, + "resources": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceRequirements", + "type": "object" + }, + "revisionHistoryLimit": { + "description": "Specifies the number of old history for the DaemonSet to retain to allow rollback.", + "type": [ + "integer", + "null" + ] + }, + "securityContext": { + "description": "[SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-worker container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecurityContext", + "type": "object" + }, + "serviceAccount": { + "type": "object", + "properties": { + "annotations": { + "description": "[Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations", + "type": "object" + }, + "create": { + "description": "Specifies whether a service account should be created.", + "type": "boolean" + }, + "name": { + "description": "Name of the service account to use. If not set and create is true, a name is generated using the fullname template", + "type": [ + "string", + "null" + ] + } + } + }, + "tolerations": { + "description": "[Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-worker pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations", + "type": "array" + }, + "updateStrategy": { + "description": "Update strategy for the nfd-worker DaemonSet. Defaults to a rolling update with `maxUnavailable: \"10%\"` so upgrades complete in a bounded number of waves on clusters of any size (the Kubernetes default of `maxUnavailable: 1` rolls one node at a time and makes Helm wait and Flux HelmRelease timeouts likely on large clusters). nfd-worker is stateless and node labels persist while a worker pod restarts, so a faster roll is safe. Set `maxUnavailable: 1` to restore the Kubernetes default. To use `type: OnDelete`, also set `rollingUpdate: null` (Helm deep-merges maps). [More info](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set)", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.apps.v1.DaemonSetUpdateStrategy", + "type": "object", + "properties": { + "rollingUpdate": { + "type": "object", + "properties": { + "maxUnavailable": { + "type": [ + "string", + "integer" + ] + } + } + } + } + } + } + } + }, + "$defs": { + "_definitions.json": { + "definitions": { + "io.k8s.api.apps.v1.DaemonSetUpdateStrategy": { + "description": "DaemonSetUpdateStrategy is a struct used to control the update strategy for a DaemonSet.", + "type": "object", + "properties": { + "rollingUpdate": { + "description": "Rolling update config params. Present only if type = \"RollingUpdate\".", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.apps.v1.RollingUpdateDaemonSet" + }, + "type": { + "description": "Type of daemon set update. Can be \"RollingUpdate\" or \"OnDelete\". Default is RollingUpdate.", + "type": "string" + } + } + }, + "io.k8s.api.apps.v1.RollingUpdateDaemonSet": { + "description": "Spec to control the desired behavior of daemon set rolling update.", + "type": "object", + "properties": { + "maxSurge": { + "description": "The maximum number of nodes with an existing available DaemonSet pod that can have an updated DaemonSet pod during during an update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxUnavailable is 0. Absolute number is calculated from percentage by rounding up to a minimum of 1. Default value is 0. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their a new pod created before the old pod is marked as deleted. The update starts by launching new pods on 30% of nodes. Once an updated pod is available (Ready for at least minReadySeconds) the old DaemonSet pod on that node is marked deleted. If the old pod becomes unavailable for any reason (Ready transitions to false, is evicted, or is drained) an updated pod is immediately created on that node without considering surge limits. Allowing surge implies the possibility that the resources consumed by the daemonset on any given node can double if the readiness check fails, and so resource intensive daemonsets should take into account that they may cause evictions during disruption.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + }, + "maxUnavailable": { + "description": "The maximum number of DaemonSet pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of total number of DaemonSet pods at the start of the update (ex: 10%). Absolute number is calculated from percentage by rounding up. This cannot be 0 if MaxSurge is 0 Default value is 1. Example: when this is set to 30%, at most 30% of the total number of nodes that should be running the daemon pod (i.e. status.desiredNumberScheduled) can have their pods stopped for an update at any given time. The update starts by stopping at most 30% of those DaemonSet pods and then brings up new DaemonSet pods in their place. Once the new pods are available, it then proceeds onto other DaemonSet pods, thus ensuring that at least 70% of original number of DaemonSet pods are available at all times during the update.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + } + } + }, + "io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource": { + "description": "Represents a Persistent Disk resource in AWS.\n\nAn AWS EBS disk must exist before mounting to a container. The disk must also be in the same AWS zone as the kubelet. An AWS EBS disk can only be mounted as read/write once. AWS EBS volumes support ownership management and SELinux relabeling.", + "type": "object", + "required": [ + "volumeID" + ], + "properties": { + "fsType": { + "description": "fsType is the filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "type": "string" + }, + "partition": { + "description": "partition is the partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty).", + "type": "integer", + "format": "int32" + }, + "readOnly": { + "description": "readOnly value true will force the readOnly setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "type": "boolean" + }, + "volumeID": { + "description": "volumeID is unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.Affinity": { + "description": "Affinity is a group of affinity scheduling rules.", + "type": "object", + "properties": { + "nodeAffinity": { + "description": "Describes node affinity scheduling rules for the pod.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NodeAffinity" + }, + "podAffinity": { + "description": "Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)).", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodAffinity" + }, + "podAntiAffinity": { + "description": "Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)).", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodAntiAffinity" + } + } + }, + "io.k8s.api.core.v1.AppArmorProfile": { + "description": "AppArmorProfile defines a pod or container's AppArmor settings.", + "type": "object", + "required": [ + "type" + ], + "properties": { + "localhostProfile": { + "description": "localhostProfile indicates a profile loaded on the node that should be used. The profile must be preconfigured on the node to work. Must match the loaded name of the profile. Must be set if and only if type is \"Localhost\".", + "type": "string" + }, + "type": { + "description": "type indicates which kind of AppArmor profile will be applied. Valid options are:\n Localhost - a profile pre-loaded on the node.\n RuntimeDefault - the container runtime's default profile.\n Unconfined - no AppArmor enforcement.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.AzureDiskVolumeSource": { + "description": "AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod.", + "type": "object", + "required": [ + "diskName", + "diskURI" + ], + "properties": { + "cachingMode": { + "description": "cachingMode is the Host Caching mode: None, Read Only, Read Write.", + "type": "string" + }, + "diskName": { + "description": "diskName is the Name of the data disk in the blob storage", + "type": "string" + }, + "diskURI": { + "description": "diskURI is the URI of data disk in the blob storage", + "type": "string" + }, + "fsType": { + "description": "fsType is Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "kind": { + "description": "kind expected values are Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared", + "type": "string" + }, + "readOnly": { + "description": "readOnly Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.AzureFileVolumeSource": { + "description": "AzureFile represents an Azure File Service mount on the host and bind mount to the pod.", + "type": "object", + "required": [ + "secretName", + "shareName" + ], + "properties": { + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretName": { + "description": "secretName is the name of secret that contains Azure Storage Account Name and Key", + "type": "string" + }, + "shareName": { + "description": "shareName is the azure share Name", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.CSIVolumeSource": { + "description": "Represents a source location of a volume to mount, managed by an external CSI driver", + "type": "object", + "required": [ + "driver" + ], + "properties": { + "driver": { + "description": "driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster.", + "type": "string" + }, + "fsType": { + "description": "fsType to mount. Ex. \"ext4\", \"xfs\", \"ntfs\". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply.", + "type": "string" + }, + "nodePublishSecretRef": { + "description": "nodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "readOnly": { + "description": "readOnly specifies a read-only configuration for the volume. Defaults to false (read/write).", + "type": "boolean" + }, + "volumeAttributes": { + "description": "volumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values.", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "io.k8s.api.core.v1.Capabilities": { + "description": "Adds and removes POSIX capabilities from running containers.", + "type": "object", + "properties": { + "add": { + "description": "Added capabilities", + "type": "array", + "items": { + "type": "string" + } + }, + "drop": { + "description": "Removed capabilities", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.api.core.v1.CephFSVolumeSource": { + "description": "Represents a Ceph Filesystem mount that lasts the lifetime of a pod Cephfs volumes do not support ownership management or SELinux relabeling.", + "type": "object", + "required": [ + "monitors" + ], + "properties": { + "monitors": { + "description": "monitors is Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "array", + "items": { + "type": "string" + } + }, + "path": { + "description": "path is Optional: Used as the mounted root, rather than the full Ceph tree, default is /", + "type": "string" + }, + "readOnly": { + "description": "readOnly is Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "boolean" + }, + "secretFile": { + "description": "secretFile is Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "string" + }, + "secretRef": { + "description": "secretRef is Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "user": { + "description": "user is optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.CinderVolumeSource": { + "description": "Represents a cinder volume resource in Openstack. A Cinder volume must exist before mounting to a container. The volume must also be in the same region as the kubelet. Cinder volumes support ownership management and SELinux relabeling.", + "type": "object", + "required": [ + "volumeID" + ], + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "type": "string" + }, + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "type": "boolean" + }, + "secretRef": { + "description": "secretRef is optional: points to a secret object containing parameters used to connect to OpenStack.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "volumeID": { + "description": "volumeID used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ClusterTrustBundleProjection": { + "description": "ClusterTrustBundleProjection describes how to select a set of ClusterTrustBundle objects and project their contents into the pod filesystem.", + "type": "object", + "required": [ + "path" + ], + "properties": { + "labelSelector": { + "description": "Select all ClusterTrustBundles that match this label selector. Only has effect if signerName is set. Mutually-exclusive with name. If unset, interpreted as \"match nothing\". If set but empty, interpreted as \"match everything\".", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "name": { + "description": "Select a single ClusterTrustBundle by object name. Mutually-exclusive with signerName and labelSelector.", + "type": "string" + }, + "optional": { + "description": "If true, don't block pod startup if the referenced ClusterTrustBundle(s) aren't available. If using name, then the named ClusterTrustBundle is allowed not to exist. If using signerName, then the combination of signerName and labelSelector is allowed to match zero ClusterTrustBundles.", + "type": "boolean" + }, + "path": { + "description": "Relative path from the volume root to write the bundle.", + "type": "string" + }, + "signerName": { + "description": "Select all ClusterTrustBundles that match this signer name. Mutually-exclusive with name. The contents of all selected ClusterTrustBundles will be unified and deduplicated.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ConfigMapEnvSource": { + "description": "ConfigMapEnvSource selects a ConfigMap to populate the environment variables with.\n\nThe contents of the target ConfigMap's Data field will represent the key-value pairs as environment variables.", + "type": "object", + "properties": { + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the ConfigMap must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.ConfigMapKeySelector": { + "description": "Selects a key from a ConfigMap.", + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "description": "The key to select.", + "type": "string" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the ConfigMap or its key must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.ConfigMapProjection": { + "description": "Adapts a ConfigMap into a projected volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a projected volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. Note that this is identical to a configmap volume source without the default mode.", + "type": "object", + "properties": { + "items": { + "description": "items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.KeyToPath" + } + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "optional specify whether the ConfigMap or its keys must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.ConfigMapVolumeSource": { + "description": "Adapts a ConfigMap into a volume.\n\nThe contents of the target ConfigMap's Data field will be presented in a volume as files using the keys in the Data field as the file names, unless the items element is populated with specific mappings of keys to paths. ConfigMap volumes support ownership management and SELinux relabeling.", + "type": "object", + "properties": { + "defaultMode": { + "description": "defaultMode is optional: mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "type": "integer", + "format": "int32" + }, + "items": { + "description": "items if unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.KeyToPath" + } + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "optional specify whether the ConfigMap or its keys must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.Container": { + "description": "A single application container that you want to run within a pod.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "args": { + "description": "Arguments to the entrypoint. The container image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "description": "Entrypoint array. Not executed within a shell. The container image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "List of environment variables to set in the container. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EnvVar" + } + }, + "envFrom": { + "description": "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EnvFromSource" + } + }, + "image": { + "description": "Container image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", + "type": "string" + }, + "imagePullPolicy": { + "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", + "type": "string" + }, + "lifecycle": { + "description": "Actions that the management system should take in response to container lifecycle events. Cannot be updated.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Lifecycle" + }, + "livenessProbe": { + "description": "Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Probe" + }, + "name": { + "description": "Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated.", + "type": "string" + }, + "ports": { + "description": "List of ports to expose from the container. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default \"0.0.0.0\" address inside a container will be accessible from the network. Modifying this array with strategic merge patch may corrupt the data. For more information See https://github.com/kubernetes/kubernetes/issues/108255. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerPort" + } + }, + "readinessProbe": { + "description": "Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Probe" + }, + "resizePolicy": { + "description": "Resources resize policy for the container.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerResizePolicy" + } + }, + "resources": { + "description": "Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "restartPolicy": { + "description": "RestartPolicy defines the restart behavior of individual containers in a pod. This overrides the pod-level restart policy. When this field is not specified, the restart behavior is defined by the Pod's restart policy and the container type. Additionally, setting the RestartPolicy as \"Always\" for the init container will have the following effect: this init container will be continually restarted on exit until all regular containers have terminated. Once all regular containers have completed, all init containers with restartPolicy \"Always\" will be shut down. This lifecycle differs from normal init containers and is often referred to as a \"sidecar\" container. Although this init container still starts in the init container sequence, it does not wait for the container to complete before proceeding to the next init container. Instead, the next init container starts immediately after this init container is started, or after any startupProbe has successfully completed.", + "type": "string" + }, + "restartPolicyRules": { + "description": "Represents a list of rules to be checked to determine if the container should be restarted on exit. The rules are evaluated in order. Once a rule matches a container exit condition, the remaining rules are ignored. If no rule matches the container exit condition, the Container-level restart policy determines the whether the container is restarted or not. Constraints on the rules: - At most 20 rules are allowed. - Rules can have the same action. - Identical rules are not forbidden in validations. When rules are specified, container MUST set RestartPolicy explicitly even it if matches the Pod's RestartPolicy.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerRestartRule" + } + }, + "securityContext": { + "description": "SecurityContext defines the security options the container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecurityContext" + }, + "startupProbe": { + "description": "StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod's lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Probe" + }, + "stdin": { + "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", + "type": "boolean" + }, + "stdinOnce": { + "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", + "type": "boolean" + }, + "terminationMessagePath": { + "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", + "type": "string" + }, + "terminationMessagePolicy": { + "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", + "type": "string" + }, + "tty": { + "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", + "type": "boolean" + }, + "volumeDevices": { + "description": "volumeDevices is the list of block devices to be used by the container.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VolumeDevice" + } + }, + "volumeMounts": { + "description": "Pod volumes to mount into the container's filesystem. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "workingDir": { + "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ContainerPort": { + "description": "ContainerPort represents a network port in a single container.", + "type": "object", + "required": [ + "containerPort" + ], + "properties": { + "containerPort": { + "description": "Number of port to expose on the pod's IP address. This must be a valid port number, 0 \u003c x \u003c 65536.", + "type": "integer", + "format": "int32" + }, + "hostIP": { + "description": "What host IP to bind the external port to.", + "type": "string" + }, + "hostPort": { + "description": "Number of port to expose on the host. If specified, this must be a valid port number, 0 \u003c x \u003c 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this.", + "type": "integer", + "format": "int32" + }, + "name": { + "description": "If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services.", + "type": "string" + }, + "protocol": { + "description": "Protocol for port. Must be UDP, TCP, or SCTP. Defaults to \"TCP\".", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ContainerResizePolicy": { + "description": "ContainerResizePolicy represents resource resize policy for the container.", + "type": "object", + "required": [ + "resourceName", + "restartPolicy" + ], + "properties": { + "resourceName": { + "description": "Name of the resource to which this resource resize policy applies. Supported values: cpu, memory.", + "type": "string" + }, + "restartPolicy": { + "description": "Restart policy to apply when specified resource is resized. If not specified, it defaults to NotRequired.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ContainerRestartRule": { + "description": "ContainerRestartRule describes how a container exit is handled.", + "type": "object", + "required": [ + "action" + ], + "properties": { + "action": { + "description": "Specifies the action taken on a container exit if the requirements are satisfied. The only possible value is \"Restart\" to restart the container.", + "type": "string" + }, + "exitCodes": { + "description": "Represents the exit codes to check on container exits.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerRestartRuleOnExitCodes" + } + } + }, + "io.k8s.api.core.v1.ContainerRestartRuleOnExitCodes": { + "description": "ContainerRestartRuleOnExitCodes describes the condition for handling an exited container based on its exit codes.", + "type": "object", + "required": [ + "operator" + ], + "properties": { + "operator": { + "description": "Represents the relationship between the container exit code(s) and the specified values. Possible values are: - In: the requirement is satisfied if the container exit code is in the\n set of specified values.\n- NotIn: the requirement is satisfied if the container exit code is\n not in the set of specified values.", + "type": "string" + }, + "values": { + "description": "Specifies the set of values to check for container exit codes. At most 255 elements are allowed.", + "type": "array", + "items": { + "type": "integer", + "format": "int32" + } + } + } + }, + "io.k8s.api.core.v1.DownwardAPIProjection": { + "description": "Represents downward API info for projecting into a projected volume. Note that this is identical to a downwardAPI volume source without the default mode.", + "type": "object", + "properties": { + "items": { + "description": "Items is a list of DownwardAPIVolume file", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.DownwardAPIVolumeFile" + } + } + } + }, + "io.k8s.api.core.v1.DownwardAPIVolumeFile": { + "description": "DownwardAPIVolumeFile represents information to create the file containing the pod field", + "type": "object", + "required": [ + "path" + ], + "properties": { + "fieldRef": { + "description": "Required: Selects a field of the pod: only annotations, labels, name, namespace and uid are supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ObjectFieldSelector" + }, + "mode": { + "description": "Optional: mode bits used to set permissions on this file, must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "type": "integer", + "format": "int32" + }, + "path": { + "description": "Required: Path is the relative path name of the file to be created. Must not be absolute or contain the '..' path. Must be utf-8 encoded. The first item of the relative path must not start with '..'", + "type": "string" + }, + "resourceFieldRef": { + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceFieldSelector" + } + } + }, + "io.k8s.api.core.v1.DownwardAPIVolumeSource": { + "description": "DownwardAPIVolumeSource represents a volume containing downward API info. Downward API volumes support ownership management and SELinux relabeling.", + "type": "object", + "properties": { + "defaultMode": { + "description": "Optional: mode bits to use on created files by default. Must be a Optional: mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "type": "integer", + "format": "int32" + }, + "items": { + "description": "Items is a list of downward API volume file", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.DownwardAPIVolumeFile" + } + } + } + }, + "io.k8s.api.core.v1.EmptyDirVolumeSource": { + "description": "Represents an empty directory for a pod. Empty directory volumes support ownership management and SELinux relabeling.", + "type": "object", + "properties": { + "medium": { + "description": "medium represents what type of storage medium should back this directory. The default is \"\" which means to use the node's default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", + "type": "string" + }, + "sizeLimit": { + "description": "sizeLimit is the total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + } + }, + "io.k8s.api.core.v1.EnvFromSource": { + "description": "EnvFromSource represents the source of a set of ConfigMaps or Secrets", + "type": "object", + "properties": { + "configMapRef": { + "description": "The ConfigMap to select from", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ConfigMapEnvSource" + }, + "prefix": { + "description": "Optional text to prepend to the name of each environment variable. May consist of any printable ASCII characters except '='.", + "type": "string" + }, + "secretRef": { + "description": "The Secret to select from", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecretEnvSource" + } + } + }, + "io.k8s.api.core.v1.EnvVar": { + "description": "EnvVar represents an environment variable present in a Container.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "Name of the environment variable. May consist of any printable ASCII characters except '='.", + "type": "string" + }, + "value": { + "description": "Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to \"\".", + "type": "string" + }, + "valueFrom": { + "description": "Source for the environment variable's value. Cannot be used if value is not empty.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EnvVarSource" + } + } + }, + "io.k8s.api.core.v1.EnvVarSource": { + "description": "EnvVarSource represents a source for the value of an EnvVar.", + "type": "object", + "properties": { + "configMapKeyRef": { + "description": "Selects a key of a ConfigMap.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ConfigMapKeySelector" + }, + "fieldRef": { + "description": "Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['\u003cKEY\u003e']`, `metadata.annotations['\u003cKEY\u003e']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ObjectFieldSelector" + }, + "fileKeyRef": { + "description": "FileKeyRef selects a key of the env file. Requires the EnvFiles feature gate to be enabled.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.FileKeySelector" + }, + "resourceFieldRef": { + "description": "Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceFieldSelector" + }, + "secretKeyRef": { + "description": "Selects a key of a secret in the pod's namespace", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecretKeySelector" + } + } + }, + "io.k8s.api.core.v1.EphemeralContainer": { + "description": "An EphemeralContainer is a temporary container that you may add to an existing Pod for user-initiated activities such as debugging. Ephemeral containers have no resource or scheduling guarantees, and they will not be restarted when they exit or when a Pod is removed or restarted. The kubelet may evict a Pod if an ephemeral container causes the Pod to exceed its resource allocation.\n\nTo add an ephemeral container, use the ephemeralcontainers subresource of an existing Pod. Ephemeral containers may not be removed or restarted.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "args": { + "description": "Arguments to the entrypoint. The image's CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + "type": "array", + "items": { + "type": "string" + } + }, + "command": { + "description": "Entrypoint array. Not executed within a shell. The image's ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container's environment. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. \"$$(VAR_NAME)\" will produce the string literal \"$(VAR_NAME)\". Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell", + "type": "array", + "items": { + "type": "string" + } + }, + "env": { + "description": "List of environment variables to set in the container. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EnvVar" + } + }, + "envFrom": { + "description": "List of sources to populate environment variables in the container. The keys defined within a source may consist of any printable ASCII characters except '='. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EnvFromSource" + } + }, + "image": { + "description": "Container image name. More info: https://kubernetes.io/docs/concepts/containers/images", + "type": "string" + }, + "imagePullPolicy": { + "description": "Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images", + "type": "string" + }, + "lifecycle": { + "description": "Lifecycle is not allowed for ephemeral containers.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Lifecycle" + }, + "livenessProbe": { + "description": "Probes are not allowed for ephemeral containers.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Probe" + }, + "name": { + "description": "Name of the ephemeral container specified as a DNS_LABEL. This name must be unique among all containers, init containers and ephemeral containers.", + "type": "string" + }, + "ports": { + "description": "Ports are not allowed for ephemeral containers.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerPort" + } + }, + "readinessProbe": { + "description": "Probes are not allowed for ephemeral containers.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Probe" + }, + "resizePolicy": { + "description": "Resources resize policy for the container.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerResizePolicy" + } + }, + "resources": { + "description": "Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources already allocated to the pod.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "restartPolicy": { + "description": "Restart policy for the container to manage the restart behavior of each container within a pod. You cannot set this field on ephemeral containers.", + "type": "string" + }, + "restartPolicyRules": { + "description": "Represents a list of rules to be checked to determine if the container should be restarted on exit. You cannot set this field on ephemeral containers.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ContainerRestartRule" + } + }, + "securityContext": { + "description": "Optional: SecurityContext defines the security options the ephemeral container should be run with. If set, the fields of SecurityContext override the equivalent fields of PodSecurityContext.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecurityContext" + }, + "startupProbe": { + "description": "Probes are not allowed for ephemeral containers.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Probe" + }, + "stdin": { + "description": "Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false.", + "type": "boolean" + }, + "stdinOnce": { + "description": "Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false", + "type": "boolean" + }, + "targetContainerName": { + "description": "If set, the name of the container from PodSpec that this ephemeral container targets. The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. If not set then the ephemeral container uses the namespaces configured in the Pod spec.\n\nThe container runtime must implement support for this feature. If the runtime does not support namespace targeting then the result of setting this field is undefined.", + "type": "string" + }, + "terminationMessagePath": { + "description": "Optional: Path at which the file to which the container's termination message will be written is mounted into the container's filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.", + "type": "string" + }, + "terminationMessagePolicy": { + "description": "Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated.", + "type": "string" + }, + "tty": { + "description": "Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false.", + "type": "boolean" + }, + "volumeDevices": { + "description": "volumeDevices is the list of block devices to be used by the container.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VolumeDevice" + } + }, + "volumeMounts": { + "description": "Pod volumes to mount into the container's filesystem. Subpath mounts are not allowed for ephemeral containers. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "workingDir": { + "description": "Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.EphemeralVolumeSource": { + "description": "Represents an ephemeral volume that is handled by a normal storage driver.", + "type": "object", + "properties": { + "volumeClaimTemplate": { + "description": "Will be used to create a stand-alone PVC to provision the volume. The pod in which this EphemeralVolumeSource is embedded will be the owner of the PVC, i.e. the PVC will be deleted together with the pod. The name of the PVC will be `\u003cpod name\u003e-\u003cvolume name\u003e` where `\u003cvolume name\u003e` is the name from the `PodSpec.Volumes` array entry. Pod validation will reject the pod if the concatenated name is not valid for a PVC (for example, too long).\n\nAn existing PVC with that name that is not owned by the pod will *not* be used for the pod to avoid using an unrelated volume by mistake. Starting the pod is then blocked until the unrelated PVC is removed. If such a pre-created PVC is meant to be used by the pod, the PVC has to updated with an owner reference to the pod once the pod exists. Normally this should not be necessary, but it may be useful when manually reconstructing a broken cluster.\n\nThis field is read-only and no changes will be made by Kubernetes to the PVC after it has been created.\n\nRequired, must not be nil.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PersistentVolumeClaimTemplate" + } + } + }, + "io.k8s.api.core.v1.ExecAction": { + "description": "ExecAction describes a \"run in container\" action.", + "type": "object", + "properties": { + "command": { + "description": "Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.api.core.v1.FCVolumeSource": { + "description": "Represents a Fibre Channel volume. Fibre Channel volumes can only be mounted as read/write once. Fibre Channel volumes support ownership management and SELinux relabeling.", + "type": "object", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "lun": { + "description": "lun is Optional: FC target lun number", + "type": "integer", + "format": "int32" + }, + "readOnly": { + "description": "readOnly is Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "targetWWNs": { + "description": "targetWWNs is Optional: FC target worldwide names (WWNs)", + "type": "array", + "items": { + "type": "string" + } + }, + "wwids": { + "description": "wwids Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.api.core.v1.FileKeySelector": { + "description": "FileKeySelector selects a key of the env file.", + "type": "object", + "required": [ + "volumeName", + "path", + "key" + ], + "properties": { + "key": { + "description": "The key within the env file. An invalid key will prevent the pod from starting. The keys defined within a source may consist of any printable ASCII characters except '='. During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters.", + "type": "string" + }, + "optional": { + "description": "Specify whether the file or its key must be defined. If the file or key does not exist, then the env var is not published. If optional is set to true and the specified key does not exist, the environment variable will not be set in the Pod's containers.\n\nIf optional is set to false and the specified key does not exist, an error will be returned during Pod creation.", + "type": "boolean" + }, + "path": { + "description": "The path within the volume from which to select the file. Must be relative and may not contain the '..' path or start with '..'.", + "type": "string" + }, + "volumeName": { + "description": "The name of the volume mount containing the env file.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.FlexVolumeSource": { + "description": "FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin.", + "type": "object", + "required": [ + "driver" + ], + "properties": { + "driver": { + "description": "driver is the name of the driver to use for this volume.", + "type": "string" + }, + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". The default filesystem depends on FlexVolume script.", + "type": "string" + }, + "options": { + "description": "options is Optional: this field holds extra command options if any.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "readOnly": { + "description": "readOnly is Optional: defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretRef": { + "description": "secretRef is Optional: secretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + } + } + }, + "io.k8s.api.core.v1.FlockerVolumeSource": { + "description": "Represents a Flocker volume mounted by the Flocker agent. One and only one of datasetName and datasetUUID should be set. Flocker volumes do not support ownership management or SELinux relabeling.", + "type": "object", + "properties": { + "datasetName": { + "description": "datasetName is Name of the dataset stored as metadata -\u003e name on the dataset for Flocker should be considered as deprecated", + "type": "string" + }, + "datasetUUID": { + "description": "datasetUUID is the UUID of the dataset. This is unique identifier of a Flocker dataset", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.GCEPersistentDiskVolumeSource": { + "description": "Represents a Persistent Disk resource in Google Compute Engine.\n\nA GCE PD must exist before mounting to a container. The disk must also be in the same GCE project and zone as the kubelet. A GCE PD can only be mounted as read/write once or read-only many times. GCE PDs support ownership management and SELinux relabeling.", + "type": "object", + "required": [ + "pdName" + ], + "properties": { + "fsType": { + "description": "fsType is filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "string" + }, + "partition": { + "description": "partition is the partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as \"1\". Similarly, the volume partition for /dev/sda is \"0\" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "integer", + "format": "int32" + }, + "pdName": { + "description": "pdName is unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.GRPCAction": { + "description": "GRPCAction specifies an action involving a GRPC service.", + "type": "object", + "required": [ + "port" + ], + "properties": { + "port": { + "description": "Port number of the gRPC service. Number must be in the range 1 to 65535.", + "type": "integer", + "format": "int32" + }, + "service": { + "description": "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).\n\nIf this is not specified, the default behavior is defined by gRPC.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.GitRepoVolumeSource": { + "description": "Represents a volume that is populated with the contents of a git repository. Git repo volumes do not support ownership management. Git repo volumes support SELinux relabeling.\n\nDEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod's container.", + "type": "object", + "required": [ + "repository" + ], + "properties": { + "directory": { + "description": "directory is the target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name.", + "type": "string" + }, + "repository": { + "description": "repository is the URL", + "type": "string" + }, + "revision": { + "description": "revision is the commit hash for the specified revision.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.GlusterfsVolumeSource": { + "description": "Represents a Glusterfs mount that lasts the lifetime of a pod. Glusterfs volumes do not support ownership management or SELinux relabeling.", + "type": "object", + "required": [ + "endpoints", + "path" + ], + "properties": { + "endpoints": { + "description": "endpoints is the endpoint name that details Glusterfs topology.", + "type": "string" + }, + "path": { + "description": "path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.HTTPGetAction": { + "description": "HTTPGetAction describes an action based on HTTP Get requests.", + "type": "object", + "required": [ + "port" + ], + "properties": { + "host": { + "description": "Host name to connect to, defaults to the pod IP. You probably want to set \"Host\" in httpHeaders instead.", + "type": "string" + }, + "httpHeaders": { + "description": "Custom headers to set in the request. HTTP allows repeated headers.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.HTTPHeader" + } + }, + "path": { + "description": "Path to access on the HTTP server.", + "type": "string" + }, + "port": { + "description": "Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + }, + "scheme": { + "description": "Scheme to use for connecting to the host. Defaults to HTTP.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.HTTPHeader": { + "description": "HTTPHeader describes a custom header to be used in HTTP probes", + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "description": "The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header.", + "type": "string" + }, + "value": { + "description": "The header field value", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.HostAlias": { + "description": "HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file.", + "type": "object", + "required": [ + "ip" + ], + "properties": { + "hostnames": { + "description": "Hostnames for the above IP address.", + "type": "array", + "items": { + "type": "string" + } + }, + "ip": { + "description": "IP address of the host file entry.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.HostPathVolumeSource": { + "description": "Represents a host path mapped into a pod. Host path volumes do not support ownership management or SELinux relabeling.", + "type": "object", + "required": [ + "path" + ], + "properties": { + "path": { + "description": "path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "string" + }, + "type": { + "description": "type for HostPath Volume Defaults to \"\" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ISCSIVolumeSource": { + "description": "Represents an ISCSI disk. ISCSI volumes can only be mounted as read/write once. ISCSI volumes support ownership management and SELinux relabeling.", + "type": "object", + "required": [ + "targetPortal", + "iqn", + "lun" + ], + "properties": { + "chapAuthDiscovery": { + "description": "chapAuthDiscovery defines whether support iSCSI Discovery CHAP authentication", + "type": "boolean" + }, + "chapAuthSession": { + "description": "chapAuthSession defines whether support iSCSI Session CHAP authentication", + "type": "boolean" + }, + "fsType": { + "description": "fsType is the filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi", + "type": "string" + }, + "initiatorName": { + "description": "initiatorName is the custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface \u003ctarget portal\u003e:\u003cvolume name\u003e will be created for the connection.", + "type": "string" + }, + "iqn": { + "description": "iqn is the target iSCSI Qualified Name.", + "type": "string" + }, + "iscsiInterface": { + "description": "iscsiInterface is the interface Name that uses an iSCSI transport. Defaults to 'default' (tcp).", + "type": "string" + }, + "lun": { + "description": "lun represents iSCSI Target Lun number.", + "type": "integer", + "format": "int32" + }, + "portals": { + "description": "portals is the iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", + "type": "array", + "items": { + "type": "string" + } + }, + "readOnly": { + "description": "readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false.", + "type": "boolean" + }, + "secretRef": { + "description": "secretRef is the CHAP Secret for iSCSI target and initiator authentication", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "targetPortal": { + "description": "targetPortal is iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260).", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ImageVolumeSource": { + "description": "ImageVolumeSource represents a image volume resource.", + "type": "object", + "properties": { + "pullPolicy": { + "description": "Policy for pulling OCI objects. Possible values are: Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise.", + "type": "string" + }, + "reference": { + "description": "Required: Image or artifact reference to be used. Behaves in the same way as pod.spec.containers[*].image. Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.KeyToPath": { + "description": "Maps a string key to a path within a volume.", + "type": "object", + "required": [ + "key", + "path" + ], + "properties": { + "key": { + "description": "key is the key to project.", + "type": "string" + }, + "mode": { + "description": "mode is Optional: mode bits used to set permissions on this file. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "type": "integer", + "format": "int32" + }, + "path": { + "description": "path is the relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.Lifecycle": { + "description": "Lifecycle describes actions that the management system should take in response to container lifecycle events. For the PostStart and PreStop lifecycle handlers, management of the container blocks until the action is complete, unless the container process fails, in which case the handler is aborted.", + "type": "object", + "properties": { + "postStart": { + "description": "PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LifecycleHandler" + }, + "preStop": { + "description": "PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The Pod's termination grace period countdown begins before the PreStop hook is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod's termination grace period (unless delayed by finalizers). Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LifecycleHandler" + }, + "stopSignal": { + "description": "StopSignal defines which signal will be sent to a container when it is being stopped. If not specified, the default is defined by the container runtime in use. StopSignal can only be set for Pods with a non-empty .spec.os.name", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.LifecycleHandler": { + "description": "LifecycleHandler defines a specific action that should be taken in a lifecycle hook. One and only one of the fields, except TCPSocket must be specified.", + "type": "object", + "properties": { + "exec": { + "description": "Exec specifies a command to execute in the container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ExecAction" + }, + "httpGet": { + "description": "HTTPGet specifies an HTTP GET request to perform.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.HTTPGetAction" + }, + "sleep": { + "description": "Sleep represents a duration that the container should sleep.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SleepAction" + }, + "tcpSocket": { + "description": "Deprecated. TCPSocket is NOT supported as a LifecycleHandler and kept for backward compatibility. There is no validation of this field and lifecycle hooks will fail at runtime when it is specified.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.TCPSocketAction" + } + } + }, + "io.k8s.api.core.v1.LocalObjectReference": { + "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.", + "type": "object", + "properties": { + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.NFSVolumeSource": { + "description": "Represents an NFS mount that lasts the lifetime of a pod. NFS volumes do not support ownership management or SELinux relabeling.", + "type": "object", + "required": [ + "server", + "path" + ], + "properties": { + "path": { + "description": "path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "type": "boolean" + }, + "server": { + "description": "server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.NodeAffinity": { + "description": "Node affinity is a group of node affinity scheduling rules.", + "type": "object", + "properties": { + "preferredDuringSchedulingIgnoredDuringExecution": { + "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PreferredSchedulingTerm" + } + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NodeSelector" + } + } + }, + "io.k8s.api.core.v1.NodeSelector": { + "description": "A node selector represents the union of the results of one or more label queries over a set of nodes; that is, it represents the OR of the selectors represented by the node selector terms.", + "type": "object", + "required": [ + "nodeSelectorTerms" + ], + "properties": { + "nodeSelectorTerms": { + "description": "Required. A list of node selector terms. The terms are ORed.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NodeSelectorTerm" + } + } + } + }, + "io.k8s.api.core.v1.NodeSelectorRequirement": { + "description": "A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", + "type": "object", + "required": [ + "key", + "operator" + ], + "properties": { + "key": { + "description": "The label key that the selector applies to.", + "type": "string" + }, + "operator": { + "description": "Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.", + "type": "string" + }, + "values": { + "description": "An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.api.core.v1.NodeSelectorTerm": { + "description": "A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm.", + "type": "object", + "properties": { + "matchExpressions": { + "description": "A list of node selector requirements by node's labels.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NodeSelectorRequirement" + } + }, + "matchFields": { + "description": "A list of node selector requirements by node's fields.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NodeSelectorRequirement" + } + } + } + }, + "io.k8s.api.core.v1.ObjectFieldSelector": { + "description": "ObjectFieldSelector selects an APIVersioned field of an object.", + "type": "object", + "required": [ + "fieldPath" + ], + "properties": { + "apiVersion": { + "description": "Version of the schema the FieldPath is written in terms of, defaults to \"v1\".", + "type": "string" + }, + "fieldPath": { + "description": "Path of the field to select in the specified API version.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PersistentVolumeClaimSpec": { + "description": "PersistentVolumeClaimSpec describes the common attributes of storage devices and allows a Source for provider-specific attributes", + "type": "object", + "properties": { + "accessModes": { + "description": "accessModes contains the desired access modes the volume should have. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1", + "type": "array", + "items": { + "type": "string" + } + }, + "dataSource": { + "description": "dataSource field can be used to specify either: * An existing VolumeSnapshot object (snapshot.storage.k8s.io/VolumeSnapshot) * An existing PVC (PersistentVolumeClaim) If the provisioner or an external controller can support the specified data source, it will create a new volume based on the contents of the specified data source. When the AnyVolumeDataSource feature gate is enabled, dataSource contents will be copied to dataSourceRef, and dataSourceRef contents will be copied to dataSource when dataSourceRef.namespace is not specified. If the namespace is specified, then dataSourceRef will not be copied to dataSource.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.TypedLocalObjectReference" + }, + "dataSourceRef": { + "description": "dataSourceRef specifies the object from which to populate the volume with data, if a non-empty volume is desired. This may be any object from a non-empty API group (non core object) or a PersistentVolumeClaim object. When this field is specified, volume binding will only succeed if the type of the specified object matches some installed volume populator or dynamic provisioner. This field will replace the functionality of the dataSource field and as such if both fields are non-empty, they must have the same value. For backwards compatibility, when namespace isn't specified in dataSourceRef, both fields (dataSource and dataSourceRef) will be set to the same value automatically if one of them is empty and the other is non-empty. When namespace is specified in dataSourceRef, dataSource isn't set to the same value and must be empty. There are three important differences between dataSource and dataSourceRef: * While dataSource only allows two specific types of objects, dataSourceRef\n allows any non-core object, as well as PersistentVolumeClaim objects.\n* While dataSource ignores disallowed values (dropping them), dataSourceRef\n preserves all values, and generates an error if a disallowed value is\n specified.\n* While dataSource only allows local objects, dataSourceRef allows objects\n in any namespaces.\n(Beta) Using this field requires the AnyVolumeDataSource feature gate to be enabled. (Alpha) Using the namespace field of dataSourceRef requires the CrossNamespaceVolumeDataSource feature gate to be enabled.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.TypedObjectReference" + }, + "resources": { + "description": "resources represents the minimum resources the volume should have. If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VolumeResourceRequirements" + }, + "selector": { + "description": "selector is a label query over volumes to consider for binding.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "storageClassName": { + "description": "storageClassName is the name of the StorageClass required by the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1", + "type": "string" + }, + "volumeAttributesClassName": { + "description": "volumeAttributesClassName may be used to set the VolumeAttributesClass used by this claim. If specified, the CSI driver will create or update the volume with the attributes defined in the corresponding VolumeAttributesClass. This has a different purpose than storageClassName, it can be changed after the claim is created. An empty string or nil value indicates that no VolumeAttributesClass will be applied to the claim. If the claim enters an Infeasible error state, this field can be reset to its previous value (including nil) to cancel the modification. If the resource referred to by volumeAttributesClass does not exist, this PersistentVolumeClaim will be set to a Pending state, as reflected by the modifyVolumeStatus field, until such as a resource exists. More info: https://kubernetes.io/docs/concepts/storage/volume-attributes-classes/", + "type": "string" + }, + "volumeMode": { + "description": "volumeMode defines what type of volume is required by the claim. Value of Filesystem is implied when not included in claim spec.", + "type": "string" + }, + "volumeName": { + "description": "volumeName is the binding reference to the PersistentVolume backing this claim.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PersistentVolumeClaimTemplate": { + "description": "PersistentVolumeClaimTemplate is used to produce PersistentVolumeClaim objects as part of an EphemeralVolumeSource.", + "type": "object", + "required": [ + "spec" + ], + "properties": { + "metadata": { + "description": "May contain labels and annotations that will be copied into the PVC when creating it. No other fields are allowed and will be rejected during validation.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" + }, + "spec": { + "description": "The specification for the PersistentVolumeClaim. The entire content is copied unchanged into the PVC that gets created from this template. The same fields as in a PersistentVolumeClaim are also valid here.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PersistentVolumeClaimSpec" + } + } + }, + "io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource": { + "description": "PersistentVolumeClaimVolumeSource references the user's PVC in the same namespace. This volume finds the bound PV and mounts that volume for the pod. A PersistentVolumeClaimVolumeSource is, essentially, a wrapper around another type of volume that is owned by someone else (the system).", + "type": "object", + "required": [ + "claimName" + ], + "properties": { + "claimName": { + "description": "claimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", + "type": "string" + }, + "readOnly": { + "description": "readOnly Will force the ReadOnly setting in VolumeMounts. Default false.", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource": { + "description": "Represents a Photon Controller persistent disk resource.", + "type": "object", + "required": [ + "pdID" + ], + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "pdID": { + "description": "pdID is the ID that identifies Photon Controller persistent disk", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodAffinity": { + "description": "Pod affinity is a group of inter pod affinity scheduling rules.", + "type": "object", + "properties": { + "preferredDuringSchedulingIgnoredDuringExecution": { + "description": "The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding \"weight\" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.WeightedPodAffinityTerm" + } + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + "description": "If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodAffinityTerm" + } + } + } + }, + "io.k8s.api.core.v1.PodAffinityTerm": { + "description": "Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key \u003ctopologyKey\u003e matches that of any node on which a pod of the set of pods is running", + "type": "object", + "required": [ + "topologyKey" + ], + "properties": { + "labelSelector": { + "description": "A label query over a set of resources, in this case pods. If it's null, this PodAffinityTerm matches with no Pods.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "matchLabelKeys": { + "description": "MatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key in (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both matchLabelKeys and labelSelector. Also, matchLabelKeys cannot be set when labelSelector isn't set.", + "type": "array", + "items": { + "type": "string" + } + }, + "mismatchLabelKeys": { + "description": "MismatchLabelKeys is a set of pod label keys to select which pods will be taken into consideration. The keys are used to lookup values from the incoming pod labels, those key-value labels are merged with `labelSelector` as `key notin (value)` to select the group of existing pods which pods will be taken into consideration for the incoming pod's pod (anti) affinity. Keys that don't exist in the incoming pod labels will be ignored. The default value is empty. The same key is forbidden to exist in both mismatchLabelKeys and labelSelector. Also, mismatchLabelKeys cannot be set when labelSelector isn't set.", + "type": "array", + "items": { + "type": "string" + } + }, + "namespaceSelector": { + "description": "A label query over the set of namespaces that the term applies to. The term is applied to the union of the namespaces selected by this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means \"this pod's namespace\". An empty selector ({}) matches all namespaces.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "namespaces": { + "description": "namespaces specifies a static list of namespace names that the term applies to. The term is applied to the union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means \"this pod's namespace\".", + "type": "array", + "items": { + "type": "string" + } + }, + "topologyKey": { + "description": "This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodAntiAffinity": { + "description": "Pod anti affinity is a group of inter pod anti affinity scheduling rules.", + "type": "object", + "properties": { + "preferredDuringSchedulingIgnoredDuringExecution": { + "description": "The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and subtracting \"weight\" from the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.WeightedPodAffinityTerm" + } + }, + "requiredDuringSchedulingIgnoredDuringExecution": { + "description": "If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodAffinityTerm" + } + } + } + }, + "io.k8s.api.core.v1.PodCertificateProjection": { + "description": "PodCertificateProjection provides a private key and X.509 certificate in the pod filesystem.", + "type": "object", + "required": [ + "signerName", + "keyType" + ], + "properties": { + "certificateChainPath": { + "description": "Write the certificate chain at this path in the projected volume.\n\nMost applications should use credentialBundlePath. When using keyPath and certificateChainPath, your application needs to check that the key and leaf certificate are consistent, because it is possible to read the files mid-rotation.", + "type": "string" + }, + "credentialBundlePath": { + "description": "Write the credential bundle at this path in the projected volume.\n\nThe credential bundle is a single file that contains multiple PEM blocks. The first PEM block is a PRIVATE KEY block, containing a PKCS#8 private key.\n\nThe remaining blocks are CERTIFICATE blocks, containing the issued certificate chain from the signer (leaf and any intermediates).\n\nUsing credentialBundlePath lets your Pod's application code make a single atomic read that retrieves a consistent key and certificate chain. If you project them to separate files, your application code will need to additionally check that the leaf certificate was issued to the key.", + "type": "string" + }, + "keyPath": { + "description": "Write the key at this path in the projected volume.\n\nMost applications should use credentialBundlePath. When using keyPath and certificateChainPath, your application needs to check that the key and leaf certificate are consistent, because it is possible to read the files mid-rotation.", + "type": "string" + }, + "keyType": { + "description": "The type of keypair Kubelet will generate for the pod.\n\nValid values are \"RSA3072\", \"RSA4096\", \"ECDSAP256\", \"ECDSAP384\", \"ECDSAP521\", and \"ED25519\".", + "type": "string" + }, + "maxExpirationSeconds": { + "description": "maxExpirationSeconds is the maximum lifetime permitted for the certificate.\n\nKubelet copies this value verbatim into the PodCertificateRequests it generates for this projection.\n\nIf omitted, kube-apiserver will set it to 86400(24 hours). kube-apiserver will reject values shorter than 3600 (1 hour). The maximum allowable value is 7862400 (91 days).\n\nThe signer implementation is then free to issue a certificate with any lifetime *shorter* than MaxExpirationSeconds, but no shorter than 3600 seconds (1 hour). This constraint is enforced by kube-apiserver. `kubernetes.io` signers will never issue certificates with a lifetime longer than 24 hours.", + "type": "integer", + "format": "int32" + }, + "signerName": { + "description": "Kubelet's generated CSRs will be addressed to this signer.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodDNSConfig": { + "description": "PodDNSConfig defines the DNS parameters of a pod in addition to those generated from DNSPolicy.", + "type": "object", + "properties": { + "nameservers": { + "description": "A list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. Duplicated nameservers will be removed.", + "type": "array", + "items": { + "type": "string" + } + }, + "options": { + "description": "A list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Duplicated entries will be removed. Resolution options given in Options will override those that appear in the base DNSPolicy.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodDNSConfigOption" + } + }, + "searches": { + "description": "A list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. Duplicated search paths will be removed.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.api.core.v1.PodDNSConfigOption": { + "description": "PodDNSConfigOption defines DNS resolver options of a pod.", + "type": "object", + "properties": { + "name": { + "description": "Name is this DNS resolver option's name. Required.", + "type": "string" + }, + "value": { + "description": "Value is this DNS resolver option's value.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodOS": { + "description": "PodOS defines the OS parameters of a pod.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "Name is the name of the operating system. The currently supported values are linux and windows. Additional value may be defined in future and can be one of: https://github.com/opencontainers/runtime-spec/blob/master/config.md#platform-specific-configuration Clients should expect to handle additional values and treat unrecognized values in this field as os: null", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodReadinessGate": { + "description": "PodReadinessGate contains the reference to a pod condition", + "type": "object", + "required": [ + "conditionType" + ], + "properties": { + "conditionType": { + "description": "ConditionType refers to a condition in the pod's condition list with matching type.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodResourceClaim": { + "description": "PodResourceClaim references exactly one ResourceClaim, either directly or by naming a ResourceClaimTemplate which is then turned into a ResourceClaim for the pod.\n\nIt adds a name to it that uniquely identifies the ResourceClaim inside the Pod. Containers that need access to the ResourceClaim reference it with this name.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "Name uniquely identifies this resource claim inside the pod. This must be a DNS_LABEL.", + "type": "string" + }, + "resourceClaimName": { + "description": "ResourceClaimName is the name of a ResourceClaim object in the same namespace as this pod.\n\nExactly one of ResourceClaimName and ResourceClaimTemplateName must be set.", + "type": "string" + }, + "resourceClaimTemplateName": { + "description": "ResourceClaimTemplateName is the name of a ResourceClaimTemplate object in the same namespace as this pod.\n\nThe template will be used to create a new ResourceClaim, which will be bound to this pod. When this pod is deleted, the ResourceClaim will also be deleted. The pod name and resource name, along with a generated component, will be used to form a unique name for the ResourceClaim, which will be recorded in pod.status.resourceClaimStatuses.\n\nThis field is immutable and no changes will be made to the corresponding ResourceClaim by the control plane after creating the ResourceClaim.\n\nExactly one of ResourceClaimName and ResourceClaimTemplateName must be set.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodSchedulingGate": { + "description": "PodSchedulingGate is associated to a Pod to guard its scheduling.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "Name of the scheduling gate. Each scheduling gate must have a unique name field.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PodSecurityContext": { + "description": "PodSecurityContext holds pod-level security attributes and common container settings. Some fields are also present in container.securityContext. Field values of container.securityContext take precedence over field values of PodSecurityContext.", + "type": "object", + "properties": { + "appArmorProfile": { + "description": "appArmorProfile is the AppArmor options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.AppArmorProfile" + }, + "fsGroup": { + "description": "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod:\n\n1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw----\n\nIf unset, the Kubelet will not modify the ownership and permissions of any volume. Note that this field cannot be set when spec.os.name is windows.", + "type": "integer", + "format": "int64" + }, + "fsGroupChangePolicy": { + "description": "fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are \"OnRootMismatch\" and \"Always\". If not specified, \"Always\" is used. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "runAsGroup": { + "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.", + "type": "integer", + "format": "int64" + }, + "runAsNonRoot": { + "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", + "type": "boolean" + }, + "runAsUser": { + "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.", + "type": "integer", + "format": "int64" + }, + "seLinuxChangePolicy": { + "description": "seLinuxChangePolicy defines how the container's SELinux label is applied to all volumes used by the Pod. It has no effect on nodes that do not support SELinux or to volumes does not support SELinux. Valid values are \"MountOption\" and \"Recursive\".\n\n\"Recursive\" means relabeling of all files on all Pod volumes by the container runtime. This may be slow for large volumes, but allows mixing privileged and unprivileged Pods sharing the same volume on the same node.\n\n\"MountOption\" mounts all eligible Pod volumes with `-o context` mount option. This requires all Pods that share the same volume to use the same SELinux label. It is not possible to share the same volume among privileged and unprivileged Pods. Eligible volumes are in-tree FibreChannel and iSCSI volumes, and all CSI volumes whose CSI driver announces SELinux support by setting spec.seLinuxMount: true in their CSIDriver instance. Other volumes are always re-labelled recursively. \"MountOption\" value is allowed only when SELinuxMount feature gate is enabled.\n\nIf not specified and SELinuxMount feature gate is enabled, \"MountOption\" is used. If not specified and SELinuxMount feature gate is disabled, \"MountOption\" is used for ReadWriteOncePod volumes and \"Recursive\" for all other volumes.\n\nThis field affects only Pods that have SELinux label set, either in PodSecurityContext or in SecurityContext of all containers.\n\nAll Pods that use the same volume should use the same seLinuxChangePolicy, otherwise some pods can get stuck in ContainerCreating state. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "seLinuxOptions": { + "description": "The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SELinuxOptions" + }, + "seccompProfile": { + "description": "The seccomp options to use by the containers in this pod. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SeccompProfile" + }, + "supplementalGroups": { + "description": "A list of groups applied to the first process run in each container, in addition to the container's primary GID and fsGroup (if specified). If the SupplementalGroupsPolicy feature is enabled, the supplementalGroupsPolicy field determines whether these are in addition to or instead of any group memberships defined in the container image. If unspecified, no additional groups are added, though group memberships defined in the container image may still be used, depending on the supplementalGroupsPolicy field. Note that this field cannot be set when spec.os.name is windows.", + "type": "array", + "items": { + "type": "integer", + "format": "int64" + } + }, + "supplementalGroupsPolicy": { + "description": "Defines how supplemental groups of the first container processes are calculated. Valid values are \"Merge\" and \"Strict\". If not specified, \"Merge\" is used. (Alpha) Using the field requires the SupplementalGroupsPolicy feature gate to be enabled and the container runtime must implement support for this feature. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "sysctls": { + "description": "Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. Note that this field cannot be set when spec.os.name is windows.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Sysctl" + } + }, + "windowsOptions": { + "description": "The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.WindowsSecurityContextOptions" + } + } + }, + "io.k8s.api.core.v1.PodSpec": { + "description": "PodSpec is a description of a pod.", + "type": "object", + "required": [ + "containers" + ], + "properties": { + "activeDeadlineSeconds": { + "description": "Optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers. Value must be a positive integer.", + "type": "integer", + "format": "int64" + }, + "affinity": { + "description": "If specified, the pod's scheduling constraints", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Affinity" + }, + "automountServiceAccountToken": { + "description": "AutomountServiceAccountToken indicates whether a service account token should be automatically mounted.", + "type": "boolean" + }, + "containers": { + "description": "List of containers belonging to the pod. Containers cannot currently be added or removed. There must be at least one container in a Pod. Cannot be updated.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container" + } + }, + "dnsConfig": { + "description": "Specifies the DNS parameters of a pod. Parameters specified here will be merged to the generated DNS configuration based on DNSPolicy.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodDNSConfig" + }, + "dnsPolicy": { + "description": "Set DNS policy for the pod. Defaults to \"ClusterFirst\". Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'.", + "type": "string" + }, + "enableServiceLinks": { + "description": "EnableServiceLinks indicates whether information about services should be injected into pod's environment variables, matching the syntax of Docker links. Optional: Defaults to true.", + "type": "boolean" + }, + "ephemeralContainers": { + "description": "List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing pod to perform user-initiated actions such as debugging. This list cannot be specified when creating a pod, and it cannot be modified by updating the pod spec. In order to add an ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EphemeralContainer" + } + }, + "hostAliases": { + "description": "HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts file if specified.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.HostAlias" + } + }, + "hostIPC": { + "description": "Use the host's ipc namespace. Optional: Default to false.", + "type": "boolean" + }, + "hostNetwork": { + "description": "Host networking requested for this pod. Use the host's network namespace. When using HostNetwork you should specify ports so the scheduler is aware. When `hostNetwork` is true, specified `hostPort` fields in port definitions must match `containerPort`, and unspecified `hostPort` fields in port definitions are defaulted to match `containerPort`. Default to false.", + "type": "boolean" + }, + "hostPID": { + "description": "Use the host's pid namespace. Optional: Default to false.", + "type": "boolean" + }, + "hostUsers": { + "description": "Use the host's user namespace. Optional: Default to true. If set to true or not present, the pod will be run in the host user namespace, useful for when the pod needs a feature only available to the host user namespace, such as loading a kernel module with CAP_SYS_MODULE. When set to false, a new userns is created for the pod. Setting false is useful for mitigating container breakout vulnerabilities even allowing users to run their containers as root without actually having root privileges on the host. This field is alpha-level and is only honored by servers that enable the UserNamespacesSupport feature.", + "type": "boolean" + }, + "hostname": { + "description": "Specifies the hostname of the Pod If not specified, the pod's hostname will be set to a system-defined value.", + "type": "string" + }, + "hostnameOverride": { + "description": "HostnameOverride specifies an explicit override for the pod's hostname as perceived by the pod. This field only specifies the pod's hostname and does not affect its DNS records. When this field is set to a non-empty string: - It takes precedence over the values set in `hostname` and `subdomain`. - The Pod's hostname will be set to this value. - `setHostnameAsFQDN` must be nil or set to false. - `hostNetwork` must be set to false.\n\nThis field must be a valid DNS subdomain as defined in RFC 1123 and contain at most 64 characters. Requires the HostnameOverride feature gate to be enabled.", + "type": "string" + }, + "imagePullSecrets": { + "description": "ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations for them to use. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + } + }, + "initContainers": { + "description": "List of initialization containers belonging to the pod. Init containers are executed in order prior to containers being started. If any init container fails, the pod is considered to have failed and is handled according to its restartPolicy. The name for an init container or normal container must be unique among all containers. Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. The resourceRequirements of an init container are taken into account during scheduling by finding the highest request/limit for each resource type, and then using the max of that value or the sum of the normal containers. Limits are applied to init containers in a similar fashion. Init containers cannot currently be added or removed. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Container" + } + }, + "nodeName": { + "description": "NodeName indicates in which node this pod is scheduled. If empty, this pod is a candidate for scheduling by the scheduler defined in schedulerName. Once this field is set, the kubelet for this node becomes responsible for the lifecycle of this pod. This field should not be used to express a desire for the pod to be scheduled on a specific node. https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodename", + "type": "string" + }, + "nodeSelector": { + "description": "NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node's labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "os": { + "description": "Specifies the OS of the containers in the pod. Some pod and container fields are restricted if this is set.\n\nIf the OS field is set to linux, the following fields must be unset: -securityContext.windowsOptions\n\nIf the OS field is set to windows, following fields must be unset: - spec.hostPID - spec.hostIPC - spec.hostUsers - spec.resources - spec.securityContext.appArmorProfile - spec.securityContext.seLinuxOptions - spec.securityContext.seccompProfile - spec.securityContext.fsGroup - spec.securityContext.fsGroupChangePolicy - spec.securityContext.sysctls - spec.shareProcessNamespace - spec.securityContext.runAsUser - spec.securityContext.runAsGroup - spec.securityContext.supplementalGroups - spec.securityContext.supplementalGroupsPolicy - spec.containers[*].securityContext.appArmorProfile - spec.containers[*].securityContext.seLinuxOptions - spec.containers[*].securityContext.seccompProfile - spec.containers[*].securityContext.capabilities - spec.containers[*].securityContext.readOnlyRootFilesystem - spec.containers[*].securityContext.privileged - spec.containers[*].securityContext.allowPrivilegeEscalation - spec.containers[*].securityContext.procMount - spec.containers[*].securityContext.runAsUser - spec.containers[*].securityContext.runAsGroup", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodOS" + }, + "overhead": { + "description": "Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/688-pod-overhead/README.md", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + }, + "preemptionPolicy": { + "description": "PreemptionPolicy is the Policy for preempting pods with lower priority. One of Never, PreemptLowerPriority. Defaults to PreemptLowerPriority if unset.", + "type": "string" + }, + "priority": { + "description": "The priority value. Various system components use this field to find the priority of the pod. When Priority Admission Controller is enabled, it prevents users from setting this field. The admission controller populates this field from PriorityClassName. The higher the value, the higher the priority.", + "type": "integer", + "format": "int32" + }, + "priorityClassName": { + "description": "If specified, indicates the pod's priority. \"system-node-critical\" and \"system-cluster-critical\" are two special keywords which indicate the highest priorities with the former being the highest priority. Any other name must be defined by creating a PriorityClass object with that name. If not specified, the pod priority will be default or zero if there is no default.", + "type": "string" + }, + "readinessGates": { + "description": "If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to \"True\" More info: https://git.k8s.io/enhancements/keps/sig-network/580-pod-readiness-gates", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodReadinessGate" + } + }, + "resourceClaims": { + "description": "ResourceClaims defines which ResourceClaims must be allocated and reserved before the Pod is allowed to start. The resources will be made available to those containers which consume them by name.\n\nThis is an alpha field and requires enabling the DynamicResourceAllocation feature gate.\n\nThis field is immutable.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodResourceClaim" + } + }, + "resources": { + "description": "Resources is the total amount of CPU and Memory resources required by all containers in the pod. It supports specifying Requests and Limits for \"cpu\", \"memory\" and \"hugepages-\" resource names only. ResourceClaims are not supported.\n\nThis field enables fine-grained control over resource allocation for the entire pod, allowing resource sharing among containers in a pod.\n\nThis is an alpha field and requires enabling the PodLevelResources feature gate.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "restartPolicy": { + "description": "Restart policy for all containers within the pod. One of Always, OnFailure, Never. In some contexts, only a subset of those values may be permitted. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy", + "type": "string" + }, + "runtimeClassName": { + "description": "RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the \"legacy\" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/585-runtime-class", + "type": "string" + }, + "schedulerName": { + "description": "If specified, the pod will be dispatched by specified scheduler. If not specified, the pod will be dispatched by default scheduler.", + "type": "string" + }, + "schedulingGates": { + "description": "SchedulingGates is an opaque list of values that if specified will block scheduling the pod. If schedulingGates is not empty, the pod will stay in the SchedulingGated state and the scheduler will not attempt to schedule the pod.\n\nSchedulingGates can only be set at pod creation time, and be removed only afterwards.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSchedulingGate" + } + }, + "securityContext": { + "description": "SecurityContext holds pod-level security attributes and common container settings. Optional: Defaults to empty. See type description for default values of each field.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodSecurityContext" + }, + "serviceAccount": { + "description": "DeprecatedServiceAccount is a deprecated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.", + "type": "string" + }, + "serviceAccountName": { + "description": "ServiceAccountName is the name of the ServiceAccount to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/", + "type": "string" + }, + "setHostnameAsFQDN": { + "description": "If true the pod's hostname will be configured as the pod's FQDN, rather than the leaf name (the default). In Linux containers, this means setting the FQDN in the hostname field of the kernel (the nodename field of struct utsname). In Windows containers, this means setting the registry value of hostname for the registry key HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\CurrentControlSet\\\\Services\\\\Tcpip\\\\Parameters to FQDN. If a pod does not have FQDN, this has no effect. Default to false.", + "type": "boolean" + }, + "shareProcessNamespace": { + "description": "Share a single process namespace between all of the containers in a pod. When this is set containers will be able to view and signal processes from other containers in the same pod, and the first process in each container will not be assigned PID 1. HostPID and ShareProcessNamespace cannot both be set. Optional: Default to false.", + "type": "boolean" + }, + "subdomain": { + "description": "If specified, the fully qualified Pod hostname will be \"\u003chostname\u003e.\u003csubdomain\u003e.\u003cpod namespace\u003e.svc.\u003ccluster domain\u003e\". If not specified, the pod will not have a domainname at all.", + "type": "string" + }, + "terminationGracePeriodSeconds": { + "description": "Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds.", + "type": "integer", + "format": "int64" + }, + "tolerations": { + "description": "If specified, the pod's tolerations.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "volumes": { + "description": "List of volumes that can be mounted by containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Volume" + } + } + } + }, + "io.k8s.api.core.v1.PortworxVolumeSource": { + "description": "PortworxVolumeSource represents a Portworx volume resource.", + "type": "object", + "required": [ + "volumeID" + ], + "properties": { + "fsType": { + "description": "fSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "volumeID": { + "description": "volumeID uniquely identifies a Portworx volume", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.PreferredSchedulingTerm": { + "description": "An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).", + "type": "object", + "required": [ + "weight", + "preference" + ], + "properties": { + "preference": { + "description": "A node selector term, associated with the corresponding weight.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NodeSelectorTerm" + }, + "weight": { + "description": "Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100.", + "type": "integer", + "format": "int32" + } + } + }, + "io.k8s.api.core.v1.Probe": { + "description": "Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic.", + "type": "object", + "properties": { + "exec": { + "description": "Exec specifies a command to execute in the container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ExecAction" + }, + "failureThreshold": { + "description": "Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1.", + "type": "integer", + "format": "int32" + }, + "grpc": { + "description": "GRPC specifies a GRPC HealthCheckRequest.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.GRPCAction" + }, + "httpGet": { + "description": "HTTPGet specifies an HTTP GET request to perform.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.HTTPGetAction" + }, + "initialDelaySeconds": { + "description": "Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "type": "integer", + "format": "int32" + }, + "periodSeconds": { + "description": "How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1.", + "type": "integer", + "format": "int32" + }, + "successThreshold": { + "description": "Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.", + "type": "integer", + "format": "int32" + }, + "tcpSocket": { + "description": "TCPSocket specifies a connection to a TCP port.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.TCPSocketAction" + }, + "terminationGracePeriodSeconds": { + "description": "Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.", + "type": "integer", + "format": "int64" + }, + "timeoutSeconds": { + "description": "Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes", + "type": "integer", + "format": "int32" + } + } + }, + "io.k8s.api.core.v1.ProjectedVolumeSource": { + "description": "Represents a projected volume source", + "type": "object", + "properties": { + "defaultMode": { + "description": "defaultMode are the mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "type": "integer", + "format": "int32" + }, + "sources": { + "description": "sources is the list of volume projections. Each entry in this list handles one source.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VolumeProjection" + } + } + } + }, + "io.k8s.api.core.v1.QuobyteVolumeSource": { + "description": "Represents a Quobyte mount that lasts the lifetime of a pod. Quobyte volumes do not support ownership management or SELinux relabeling.", + "type": "object", + "required": [ + "registry", + "volume" + ], + "properties": { + "group": { + "description": "group to map volume access to Default is no group", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false.", + "type": "boolean" + }, + "registry": { + "description": "registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes", + "type": "string" + }, + "tenant": { + "description": "tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin", + "type": "string" + }, + "user": { + "description": "user to map volume access to Defaults to serivceaccount user", + "type": "string" + }, + "volume": { + "description": "volume is a string that references an already created Quobyte volume by name.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.RBDVolumeSource": { + "description": "Represents a Rados Block Device mount that lasts the lifetime of a pod. RBD volumes support ownership management and SELinux relabeling.", + "type": "object", + "required": [ + "monitors", + "image" + ], + "properties": { + "fsType": { + "description": "fsType is the filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd", + "type": "string" + }, + "image": { + "description": "image is the rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + }, + "keyring": { + "description": "keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + }, + "monitors": { + "description": "monitors is a collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "array", + "items": { + "type": "string" + } + }, + "pool": { + "description": "pool is the rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + }, + "readOnly": { + "description": "readOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "boolean" + }, + "secretRef": { + "description": "secretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "user": { + "description": "user is the rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ResourceClaim": { + "description": "ResourceClaim references one entry in PodSpec.ResourceClaims.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "description": "Name must match the name of one entry in pod.spec.resourceClaims of the Pod where this field is used. It makes that resource available inside a container.", + "type": "string" + }, + "request": { + "description": "Request is the name chosen for a request in the referenced claim. If empty, everything from the claim is made available, otherwise only the result of this request.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ResourceFieldSelector": { + "description": "ResourceFieldSelector represents container resources (cpu, memory) and their output format", + "type": "object", + "required": [ + "resource" + ], + "properties": { + "containerName": { + "description": "Container name: required for volumes, optional for env vars", + "type": "string" + }, + "divisor": { + "description": "Specifies the output format of the exposed resources, defaults to \"1\"", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + }, + "resource": { + "description": "Required: resource to select", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ResourceRequirements": { + "description": "ResourceRequirements describes the compute resource requirements.", + "type": "object", + "properties": { + "claims": { + "description": "Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container.\n\nThis field depends on the DynamicResourceAllocation feature gate.\n\nThis field is immutable. It can only be set for containers.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ResourceClaim" + } + }, + "limits": { + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + }, + "requests": { + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + } + } + }, + "io.k8s.api.core.v1.SELinuxOptions": { + "description": "SELinuxOptions are the labels to be applied to the container", + "type": "object", + "properties": { + "level": { + "description": "Level is SELinux level label that applies to the container.", + "type": "string" + }, + "role": { + "description": "Role is a SELinux role label that applies to the container.", + "type": "string" + }, + "type": { + "description": "Type is a SELinux type label that applies to the container.", + "type": "string" + }, + "user": { + "description": "User is a SELinux user label that applies to the container.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.ScaleIOVolumeSource": { + "description": "ScaleIOVolumeSource represents a persistent ScaleIO volume", + "type": "object", + "required": [ + "gateway", + "system", + "secretRef" + ], + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Default is \"xfs\".", + "type": "string" + }, + "gateway": { + "description": "gateway is the host address of the ScaleIO API Gateway.", + "type": "string" + }, + "protectionDomain": { + "description": "protectionDomain is the name of the ScaleIO Protection Domain for the configured storage.", + "type": "string" + }, + "readOnly": { + "description": "readOnly Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretRef": { + "description": "secretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "sslEnabled": { + "description": "sslEnabled Flag enable/disable SSL communication with Gateway, default false", + "type": "boolean" + }, + "storageMode": { + "description": "storageMode indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned.", + "type": "string" + }, + "storagePool": { + "description": "storagePool is the ScaleIO Storage Pool associated with the protection domain.", + "type": "string" + }, + "system": { + "description": "system is the name of the storage system as configured in ScaleIO.", + "type": "string" + }, + "volumeName": { + "description": "volumeName is the name of a volume already created in the ScaleIO system that is associated with this volume source.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.SeccompProfile": { + "description": "SeccompProfile defines a pod/container's seccomp profile settings. Only one profile source may be set.", + "type": "object", + "required": [ + "type" + ], + "properties": { + "localhostProfile": { + "description": "localhostProfile indicates a profile defined in a file on the node should be used. The profile must be preconfigured on the node to work. Must be a descending path, relative to the kubelet's configured seccomp profile location. Must be set if type is \"Localhost\". Must NOT be set for any other type.", + "type": "string" + }, + "type": { + "description": "type indicates which kind of seccomp profile will be applied. Valid options are:\n\nLocalhost - a profile defined in a file on the node should be used. RuntimeDefault - the container runtime default profile should be used. Unconfined - no profile should be applied.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.SecretEnvSource": { + "description": "SecretEnvSource selects a Secret to populate the environment variables with.\n\nThe contents of the target Secret's Data field will represent the key-value pairs as environment variables.", + "type": "object", + "properties": { + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the Secret must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.SecretKeySelector": { + "description": "SecretKeySelector selects a key of a Secret.", + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "description": "The key of the secret to select from. Must be a valid secret key.", + "type": "string" + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "Specify whether the Secret or its key must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.SecretProjection": { + "description": "Adapts a secret into a projected volume.\n\nThe contents of the target Secret's Data field will be presented in a projected volume as files using the keys in the Data field as the file names. Note that this is identical to a secret volume source without the default mode.", + "type": "object", + "properties": { + "items": { + "description": "items if unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.KeyToPath" + } + }, + "name": { + "description": "Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "optional": { + "description": "optional field specify whether the Secret or its key must be defined", + "type": "boolean" + } + } + }, + "io.k8s.api.core.v1.SecretVolumeSource": { + "description": "Adapts a Secret into a volume.\n\nThe contents of the target Secret's Data field will be presented in a volume as files using the keys in the Data field as the file names. Secret volumes support ownership management and SELinux relabeling.", + "type": "object", + "properties": { + "defaultMode": { + "description": "defaultMode is Optional: mode bits used to set permissions on created files by default. Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.", + "type": "integer", + "format": "int32" + }, + "items": { + "description": "items If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.KeyToPath" + } + }, + "optional": { + "description": "optional field specify whether the Secret or its keys must be defined", + "type": "boolean" + }, + "secretName": { + "description": "secretName is the name of the secret in the pod's namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.SecurityContext": { + "description": "SecurityContext holds security configuration that will be applied to a container. Some fields are present in both SecurityContext and PodSecurityContext. When both are set, the values in SecurityContext take precedence.", + "type": "object", + "properties": { + "allowPrivilegeEscalation": { + "description": "AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN Note that this field cannot be set when spec.os.name is windows.", + "type": "boolean" + }, + "appArmorProfile": { + "description": "appArmorProfile is the AppArmor options to use by this container. If set, this profile overrides the pod's appArmorProfile. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.AppArmorProfile" + }, + "capabilities": { + "description": "The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.Capabilities" + }, + "privileged": { + "description": "Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. Note that this field cannot be set when spec.os.name is windows.", + "type": "boolean" + }, + "procMount": { + "description": "procMount denotes the type of proc mount to use for the containers. The default value is Default which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. Note that this field cannot be set when spec.os.name is windows.", + "type": "string" + }, + "readOnlyRootFilesystem": { + "description": "Whether this container has a read-only root filesystem. Default is false. Note that this field cannot be set when spec.os.name is windows.", + "type": "boolean" + }, + "runAsGroup": { + "description": "The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.", + "type": "integer", + "format": "int64" + }, + "runAsNonRoot": { + "description": "Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", + "type": "boolean" + }, + "runAsUser": { + "description": "The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.", + "type": "integer", + "format": "int64" + }, + "seLinuxOptions": { + "description": "The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SELinuxOptions" + }, + "seccompProfile": { + "description": "The seccomp options to use by this container. If seccomp options are provided at both the pod \u0026 container level, the container options override the pod options. Note that this field cannot be set when spec.os.name is windows.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SeccompProfile" + }, + "windowsOptions": { + "description": "The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. Note that this field cannot be set when spec.os.name is linux.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.WindowsSecurityContextOptions" + } + } + }, + "io.k8s.api.core.v1.ServiceAccountTokenProjection": { + "description": "ServiceAccountTokenProjection represents a projected service account token volume. This projection can be used to insert a service account token into the pods runtime filesystem for use against APIs (Kubernetes API Server or otherwise).", + "type": "object", + "required": [ + "path" + ], + "properties": { + "audience": { + "description": "audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver.", + "type": "string" + }, + "expirationSeconds": { + "description": "expirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes.", + "type": "integer", + "format": "int64" + }, + "path": { + "description": "path is the path relative to the mount point of the file to project the token into.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.SleepAction": { + "description": "SleepAction describes a \"sleep\" action.", + "type": "object", + "required": [ + "seconds" + ], + "properties": { + "seconds": { + "description": "Seconds is the number of seconds to sleep.", + "type": "integer", + "format": "int64" + } + } + }, + "io.k8s.api.core.v1.StorageOSVolumeSource": { + "description": "Represents a StorageOS persistent volume resource.", + "type": "object", + "properties": { + "fsType": { + "description": "fsType is the filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "readOnly": { + "description": "readOnly defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.", + "type": "boolean" + }, + "secretRef": { + "description": "secretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.LocalObjectReference" + }, + "volumeName": { + "description": "volumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace.", + "type": "string" + }, + "volumeNamespace": { + "description": "volumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to \"default\" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.Sysctl": { + "description": "Sysctl defines a kernel parameter to be set", + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "description": "Name of a property to set", + "type": "string" + }, + "value": { + "description": "Value of a property to set", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.TCPSocketAction": { + "description": "TCPSocketAction describes an action based on opening a socket", + "type": "object", + "required": [ + "port" + ], + "properties": { + "host": { + "description": "Optional: Host name to connect to, defaults to the pod IP.", + "type": "string" + }, + "port": { + "description": "Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + } + } + }, + "io.k8s.api.core.v1.Toleration": { + "description": "The pod this Toleration is attached to tolerates any taint that matches the triple \u003ckey,value,effect\u003e using the matching operator \u003coperator\u003e.", + "type": "object", + "properties": { + "effect": { + "description": "Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.", + "type": "string" + }, + "key": { + "description": "Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys.", + "type": "string" + }, + "operator": { + "description": "Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category.", + "type": "string" + }, + "tolerationSeconds": { + "description": "TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system.", + "type": "integer", + "format": "int64" + }, + "value": { + "description": "Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.TopologySpreadConstraint": { + "description": "TopologySpreadConstraint specifies how to spread matching pods among the given topology.", + "type": "object", + "required": [ + "maxSkew", + "topologyKey", + "whenUnsatisfiable" + ], + "properties": { + "labelSelector": { + "description": "LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "matchLabelKeys": { + "description": "MatchLabelKeys is a set of pod label keys to select the pods over which spreading will be calculated. The keys are used to lookup values from the incoming pod labels, those key-value labels are ANDed with labelSelector to select the group of existing pods over which spreading will be calculated for the incoming pod. The same key is forbidden to exist in both MatchLabelKeys and LabelSelector. MatchLabelKeys cannot be set when LabelSelector isn't set. Keys that don't exist in the incoming pod labels will be ignored. A null or empty list means only match against labelSelector.\n\nThis is a beta field and requires the MatchLabelKeysInPodTopologySpread feature gate to be enabled (enabled by default).", + "type": "array", + "items": { + "type": "string" + } + }, + "maxSkew": { + "description": "MaxSkew describes the degree to which pods may be unevenly distributed. When `whenUnsatisfiable=DoNotSchedule`, it is the maximum permitted difference between the number of matching pods in the target topology and the global minimum. The global minimum is the minimum number of matching pods in an eligible domain or zero if the number of eligible domains is less than MinDomains. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 2/2/1: In this case, the global minimum is 1. | zone1 | zone2 | zone3 | | P P | P P | P | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 2/2/2; scheduling it onto zone1(zone2) would make the ActualSkew(3-1) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. When `whenUnsatisfiable=ScheduleAnyway`, it is used to give higher precedence to topologies that satisfy it. It's a required field. Default value is 1 and 0 is not allowed.", + "type": "integer", + "format": "int32" + }, + "minDomains": { + "description": "MinDomains indicates a minimum number of eligible domains. When the number of eligible domains with matching topology keys is less than minDomains, Pod Topology Spread treats \"global minimum\" as 0, and then the calculation of Skew is performed. And when the number of eligible domains with matching topology keys equals or greater than minDomains, this value has no effect on scheduling. As a result, when the number of eligible domains is less than minDomains, scheduler won't schedule more than maxSkew Pods to those domains. If value is nil, the constraint behaves as if MinDomains is equal to 1. Valid values are integers greater than 0. When value is not nil, WhenUnsatisfiable must be DoNotSchedule.\n\nFor example, in a 3-zone cluster, MaxSkew is set to 2, MinDomains is set to 5 and pods with the same labelSelector spread as 2/2/2: | zone1 | zone2 | zone3 | | P P | P P | P P | The number of domains is less than 5(MinDomains), so \"global minimum\" is treated as 0. In this situation, new pod with the same labelSelector cannot be scheduled, because computed skew will be 3(3 - 0) if new Pod is scheduled to any of the three zones, it will violate MaxSkew.", + "type": "integer", + "format": "int32" + }, + "nodeAffinityPolicy": { + "description": "NodeAffinityPolicy indicates how we will treat Pod's nodeAffinity/nodeSelector when calculating pod topology spread skew. Options are: - Honor: only nodes matching nodeAffinity/nodeSelector are included in the calculations. - Ignore: nodeAffinity/nodeSelector are ignored. All nodes are included in the calculations.\n\nIf this value is nil, the behavior is equivalent to the Honor policy.", + "type": "string" + }, + "nodeTaintsPolicy": { + "description": "NodeTaintsPolicy indicates how we will treat node taints when calculating pod topology spread skew. Options are: - Honor: nodes without taints, along with tainted nodes for which the incoming pod has a toleration, are included. - Ignore: node taints are ignored. All nodes are included.\n\nIf this value is nil, the behavior is equivalent to the Ignore policy.", + "type": "string" + }, + "topologyKey": { + "description": "TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each \u003ckey, value\u003e as a \"bucket\", and try to put balanced number of pods into each bucket. We define a domain as a particular instance of a topology. Also, we define an eligible domain as a domain whose nodes meet the requirements of nodeAffinityPolicy and nodeTaintsPolicy. e.g. If TopologyKey is \"kubernetes.io/hostname\", each Node is a domain of that topology. And, if TopologyKey is \"topology.kubernetes.io/zone\", each zone is a domain of that topology. It's a required field.", + "type": "string" + }, + "whenUnsatisfiable": { + "description": "WhenUnsatisfiable indicates how to deal with a pod if it doesn't satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it. - ScheduleAnyway tells the scheduler to schedule the pod in any location,\n but giving higher precedence to topologies that would help reduce the\n skew.\nA constraint is considered \"Unsatisfiable\" for an incoming pod if and only if every possible node assignment for that pod would violate \"MaxSkew\" on some topology. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won't make it *more* imbalanced. It's a required field.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.TypedLocalObjectReference": { + "description": "TypedLocalObjectReference contains enough information to let you locate the typed referenced object inside the same namespace.", + "type": "object", + "required": [ + "kind", + "name" + ], + "properties": { + "apiGroup": { + "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", + "type": "string" + }, + "kind": { + "description": "Kind is the type of resource being referenced", + "type": "string" + }, + "name": { + "description": "Name is the name of resource being referenced", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.TypedObjectReference": { + "description": "TypedObjectReference contains enough information to let you locate the typed referenced object", + "type": "object", + "required": [ + "kind", + "name" + ], + "properties": { + "apiGroup": { + "description": "APIGroup is the group for the resource being referenced. If APIGroup is not specified, the specified Kind must be in the core API group. For any other third-party types, APIGroup is required.", + "type": "string" + }, + "kind": { + "description": "Kind is the type of resource being referenced", + "type": "string" + }, + "name": { + "description": "Name is the name of resource being referenced", + "type": "string" + }, + "namespace": { + "description": "Namespace is the namespace of resource being referenced Note that when a namespace is specified, a gateway.networking.k8s.io/ReferenceGrant object is required in the referent namespace to allow that namespace's owner to accept the reference. See the ReferenceGrant documentation for details. (Alpha) This field requires the CrossNamespaceVolumeDataSource feature gate to be enabled.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.Volume": { + "description": "Volume represents a named volume in a pod that may be accessed by any container in the pod.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "awsElasticBlockStore": { + "description": "awsElasticBlockStore represents an AWS Disk resource that is attached to a kubelet's host machine and then exposed to the pod. Deprecated: AWSElasticBlockStore is deprecated. All operations for the in-tree awsElasticBlockStore type are redirected to the ebs.csi.aws.com CSI driver. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.AWSElasticBlockStoreVolumeSource" + }, + "azureDisk": { + "description": "azureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. Deprecated: AzureDisk is deprecated. All operations for the in-tree azureDisk type are redirected to the disk.csi.azure.com CSI driver.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.AzureDiskVolumeSource" + }, + "azureFile": { + "description": "azureFile represents an Azure File Service mount on the host and bind mount to the pod. Deprecated: AzureFile is deprecated. All operations for the in-tree azureFile type are redirected to the file.csi.azure.com CSI driver.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.AzureFileVolumeSource" + }, + "cephfs": { + "description": "cephFS represents a Ceph FS mount on the host that shares a pod's lifetime. Deprecated: CephFS is deprecated and the in-tree cephfs type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.CephFSVolumeSource" + }, + "cinder": { + "description": "cinder represents a cinder volume attached and mounted on kubelets host machine. Deprecated: Cinder is deprecated. All operations for the in-tree cinder type are redirected to the cinder.csi.openstack.org CSI driver. More info: https://examples.k8s.io/mysql-cinder-pd/README.md", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.CinderVolumeSource" + }, + "configMap": { + "description": "configMap represents a configMap that should populate this volume", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ConfigMapVolumeSource" + }, + "csi": { + "description": "csi (Container Storage Interface) represents ephemeral storage that is handled by certain external CSI drivers.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.CSIVolumeSource" + }, + "downwardAPI": { + "description": "downwardAPI represents downward API about the pod that should populate this volume", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.DownwardAPIVolumeSource" + }, + "emptyDir": { + "description": "emptyDir represents a temporary directory that shares a pod's lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EmptyDirVolumeSource" + }, + "ephemeral": { + "description": "ephemeral represents a volume that is handled by a cluster storage driver. The volume's lifecycle is tied to the pod that defines it - it will be created before the pod starts, and deleted when the pod is removed.\n\nUse this if: a) the volume is only needed while the pod runs, b) features of normal volumes like restoring from snapshot or capacity\n tracking are needed,\nc) the storage driver is specified through a storage class, and d) the storage driver supports dynamic volume provisioning through\n a PersistentVolumeClaim (see EphemeralVolumeSource for more\n information on the connection between this volume type\n and PersistentVolumeClaim).\n\nUse PersistentVolumeClaim or one of the vendor-specific APIs for volumes that persist for longer than the lifecycle of an individual pod.\n\nUse CSI for light-weight local ephemeral volumes if the CSI driver is meant to be used that way - see the documentation of the driver for more information.\n\nA pod can use both types of ephemeral volumes and persistent volumes at the same time.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.EphemeralVolumeSource" + }, + "fc": { + "description": "fc represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.FCVolumeSource" + }, + "flexVolume": { + "description": "flexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. Deprecated: FlexVolume is deprecated. Consider using a CSIDriver instead.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.FlexVolumeSource" + }, + "flocker": { + "description": "flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running. Deprecated: Flocker is deprecated and the in-tree flocker type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.FlockerVolumeSource" + }, + "gcePersistentDisk": { + "description": "gcePersistentDisk represents a GCE Disk resource that is attached to a kubelet's host machine and then exposed to the pod. Deprecated: GCEPersistentDisk is deprecated. All operations for the in-tree gcePersistentDisk type are redirected to the pd.csi.storage.gke.io CSI driver. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.GCEPersistentDiskVolumeSource" + }, + "gitRepo": { + "description": "gitRepo represents a git repository at a particular revision. Deprecated: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod's container.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.GitRepoVolumeSource" + }, + "glusterfs": { + "description": "glusterfs represents a Glusterfs mount on the host that shares a pod's lifetime. Deprecated: Glusterfs is deprecated and the in-tree glusterfs type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.GlusterfsVolumeSource" + }, + "hostPath": { + "description": "hostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.HostPathVolumeSource" + }, + "image": { + "description": "image represents an OCI object (a container image or artifact) pulled and mounted on the kubelet's host machine. The volume is resolved at pod startup depending on which PullPolicy value is provided:\n\n- Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. - Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. - IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails.\n\nThe volume gets re-resolved if the pod gets deleted and recreated, which means that new remote content will become available on pod recreation. A failure to resolve or pull the image during pod startup will block containers from starting and may add significant latency. Failures will be retried using normal volume backoff and will be reported on the pod reason and message. The types of objects that may be mounted by this volume are defined by the container runtime implementation on a host machine and at minimum must include all valid types supported by the container image field. The OCI object gets mounted in a single directory (spec.containers[*].volumeMounts.mountPath) by merging the manifest layers in the same way as for container images. The volume will be mounted read-only (ro) and non-executable files (noexec). Sub path mounts for containers are not supported (spec.containers[*].volumeMounts.subpath) before 1.33. The field spec.securityContext.fsGroupChangePolicy has no effect on this volume type.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ImageVolumeSource" + }, + "iscsi": { + "description": "iscsi represents an ISCSI Disk resource that is attached to a kubelet's host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes/#iscsi", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ISCSIVolumeSource" + }, + "name": { + "description": "name of the volume. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names", + "type": "string" + }, + "nfs": { + "description": "nfs represents an NFS mount on the host that shares a pod's lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.NFSVolumeSource" + }, + "persistentVolumeClaim": { + "description": "persistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PersistentVolumeClaimVolumeSource" + }, + "photonPersistentDisk": { + "description": "photonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine. Deprecated: PhotonPersistentDisk is deprecated and the in-tree photonPersistentDisk type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PhotonPersistentDiskVolumeSource" + }, + "portworxVolume": { + "description": "portworxVolume represents a portworx volume attached and mounted on kubelets host machine. Deprecated: PortworxVolume is deprecated. All operations for the in-tree portworxVolume type are redirected to the pxd.portworx.com CSI driver when the CSIMigrationPortworx feature-gate is on.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PortworxVolumeSource" + }, + "projected": { + "description": "projected items for all in one resources secrets, configmaps, and downward API", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ProjectedVolumeSource" + }, + "quobyte": { + "description": "quobyte represents a Quobyte mount on the host that shares a pod's lifetime. Deprecated: Quobyte is deprecated and the in-tree quobyte type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.QuobyteVolumeSource" + }, + "rbd": { + "description": "rbd represents a Rados Block Device mount on the host that shares a pod's lifetime. Deprecated: RBD is deprecated and the in-tree rbd type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.RBDVolumeSource" + }, + "scaleIO": { + "description": "scaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. Deprecated: ScaleIO is deprecated and the in-tree scaleIO type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ScaleIOVolumeSource" + }, + "secret": { + "description": "secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecretVolumeSource" + }, + "storageos": { + "description": "storageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. Deprecated: StorageOS is deprecated and the in-tree storageos type is no longer supported.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.StorageOSVolumeSource" + }, + "vsphereVolume": { + "description": "vsphereVolume represents a vSphere volume attached and mounted on kubelets host machine. Deprecated: VsphereVolume is deprecated. All operations for the in-tree vsphereVolume type are redirected to the csi.vsphere.vmware.com CSI driver.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource" + } + } + }, + "io.k8s.api.core.v1.VolumeDevice": { + "description": "volumeDevice describes a mapping of a raw block device within a container.", + "type": "object", + "required": [ + "name", + "devicePath" + ], + "properties": { + "devicePath": { + "description": "devicePath is the path inside of the container that the device will be mapped to.", + "type": "string" + }, + "name": { + "description": "name must match the name of a persistentVolumeClaim in the pod", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.VolumeMount": { + "description": "VolumeMount describes a mounting of a Volume within a container.", + "type": "object", + "required": [ + "name", + "mountPath" + ], + "properties": { + "mountPath": { + "description": "Path within the container at which the volume should be mounted. Must not contain ':'.", + "type": "string" + }, + "mountPropagation": { + "description": "mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified (which defaults to None).", + "type": "string" + }, + "name": { + "description": "This must match the Name of a Volume.", + "type": "string" + }, + "readOnly": { + "description": "Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false.", + "type": "boolean" + }, + "recursiveReadOnly": { + "description": "RecursiveReadOnly specifies whether read-only mounts should be handled recursively.\n\nIf ReadOnly is false, this field has no meaning and must be unspecified.\n\nIf ReadOnly is true, and this field is set to Disabled, the mount is not made recursively read-only. If this field is set to IfPossible, the mount is made recursively read-only, if it is supported by the container runtime. If this field is set to Enabled, the mount is made recursively read-only if it is supported by the container runtime, otherwise the pod will not be started and an error will be generated to indicate the reason.\n\nIf this field is set to IfPossible or Enabled, MountPropagation must be set to None (or be unspecified, which defaults to None).\n\nIf this field is not specified, it is treated as an equivalent of Disabled.", + "type": "string" + }, + "subPath": { + "description": "Path within the volume from which the container's volume should be mounted. Defaults to \"\" (volume's root).", + "type": "string" + }, + "subPathExpr": { + "description": "Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to \"\" (volume's root). SubPathExpr and SubPath are mutually exclusive.", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.VolumeProjection": { + "description": "Projection that may be projected along with other supported volume types. Exactly one of these fields must be set.", + "type": "object", + "properties": { + "clusterTrustBundle": { + "description": "ClusterTrustBundle allows a pod to access the `.spec.trustBundle` field of ClusterTrustBundle objects in an auto-updating file.\n\nAlpha, gated by the ClusterTrustBundleProjection feature gate.\n\nClusterTrustBundle objects can either be selected by name, or by the combination of signer name and a label selector.\n\nKubelet performs aggressive normalization of the PEM contents written into the pod filesystem. Esoteric PEM features such as inter-block comments and block headers are stripped. Certificates are deduplicated. The ordering of certificates within the file is arbitrary, and Kubelet may change the order over time.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ClusterTrustBundleProjection" + }, + "configMap": { + "description": "configMap information about the configMap data to project", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ConfigMapProjection" + }, + "downwardAPI": { + "description": "downwardAPI information about the downwardAPI data to project", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.DownwardAPIProjection" + }, + "podCertificate": { + "description": "Projects an auto-rotating credential bundle (private key and certificate chain) that the pod can use either as a TLS client or server.\n\nKubelet generates a private key and uses it to send a PodCertificateRequest to the named signer. Once the signer approves the request and issues a certificate chain, Kubelet writes the key and certificate chain to the pod filesystem. The pod does not start until certificates have been issued for each podCertificate projected volume source in its spec.\n\nKubelet will begin trying to rotate the certificate at the time indicated by the signer using the PodCertificateRequest.Status.BeginRefreshAt timestamp.\n\nKubelet can write a single file, indicated by the credentialBundlePath field, or separate files, indicated by the keyPath and certificateChainPath fields.\n\nThe credential bundle is a single file in PEM format. The first PEM entry is the private key (in PKCS#8 format), and the remaining PEM entries are the certificate chain issued by the signer (typically, signers will return their certificate chain in leaf-to-root order).\n\nPrefer using the credential bundle format, since your application code can read it atomically. If you use keyPath and certificateChainPath, your application must make two separate file reads. If these coincide with a certificate rotation, it is possible that the private key and leaf certificate you read may not correspond to each other. Your application will need to check for this condition, and re-read until they are consistent.\n\nThe named signer controls chooses the format of the certificate it issues; consult the signer implementation's documentation to learn how to use the certificates it issues.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodCertificateProjection" + }, + "secret": { + "description": "secret information about the secret data to project", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.SecretProjection" + }, + "serviceAccountToken": { + "description": "serviceAccountToken is information about the serviceAccountToken data to project", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.ServiceAccountTokenProjection" + } + } + }, + "io.k8s.api.core.v1.VolumeResourceRequirements": { + "description": "VolumeResourceRequirements describes the storage resource requirements for a volume.", + "type": "object", + "properties": { + "limits": { + "description": "Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + }, + "requests": { + "description": "Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/", + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + } + } + }, + "io.k8s.api.core.v1.VsphereVirtualDiskVolumeSource": { + "description": "Represents a vSphere volume resource.", + "type": "object", + "required": [ + "volumePath" + ], + "properties": { + "fsType": { + "description": "fsType is filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. \"ext4\", \"xfs\", \"ntfs\". Implicitly inferred to be \"ext4\" if unspecified.", + "type": "string" + }, + "storagePolicyID": { + "description": "storagePolicyID is the storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName.", + "type": "string" + }, + "storagePolicyName": { + "description": "storagePolicyName is the storage Policy Based Management (SPBM) profile name.", + "type": "string" + }, + "volumePath": { + "description": "volumePath is the path that identifies vSphere volume vmdk", + "type": "string" + } + } + }, + "io.k8s.api.core.v1.WeightedPodAffinityTerm": { + "description": "The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s)", + "type": "object", + "required": [ + "weight", + "podAffinityTerm" + ], + "properties": { + "podAffinityTerm": { + "description": "Required. A pod affinity term, associated with the corresponding weight.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.core.v1.PodAffinityTerm" + }, + "weight": { + "description": "weight associated with matching the corresponding podAffinityTerm, in the range 1-100.", + "type": "integer", + "format": "int32" + } + } + }, + "io.k8s.api.core.v1.WindowsSecurityContextOptions": { + "description": "WindowsSecurityContextOptions contain Windows-specific options and credentials.", + "type": "object", + "properties": { + "gmsaCredentialSpec": { + "description": "GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field.", + "type": "string" + }, + "gmsaCredentialSpecName": { + "description": "GMSACredentialSpecName is the name of the GMSA credential spec to use.", + "type": "string" + }, + "hostProcess": { + "description": "HostProcess determines if a container should be run as a 'Host Process' container. All of a Pod's containers must have the same effective HostProcess value (it is not allowed to have a mix of HostProcess containers and non-HostProcess containers). In addition, if HostProcess is true then HostNetwork must also be set to true.", + "type": "boolean" + }, + "runAsUserName": { + "description": "The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence.", + "type": "string" + } + } + }, + "io.k8s.api.networking.v1.IPBlock": { + "description": "IPBlock describes a particular CIDR (Ex. \"192.168.1.0/24\",\"2001:db8::/64\") that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The except entry describes CIDRs that should not be included within this rule.", + "type": "object", + "required": [ + "cidr" + ], + "properties": { + "cidr": { + "description": "cidr is a string representing the IPBlock Valid examples are \"192.168.1.0/24\" or \"2001:db8::/64\"", + "type": "string" + }, + "except": { + "description": "except is a slice of CIDRs that should not be included within an IPBlock Valid examples are \"192.168.1.0/24\" or \"2001:db8::/64\" Except values will be rejected if they are outside the cidr range", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.api.networking.v1.NetworkPolicyEgressRule": { + "description": "NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. This type is beta-level in 1.8", + "type": "object", + "properties": { + "ports": { + "description": "ports is a list of destination ports for outgoing traffic. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + } + }, + "to": { + "description": "to is a list of destinations for outgoing traffic of pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all destinations (traffic not restricted by destination). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the to list.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + } + } + } + }, + "io.k8s.api.networking.v1.NetworkPolicyIngressRule": { + "description": "NetworkPolicyIngressRule describes a particular set of traffic that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and from.", + "type": "object", + "properties": { + "from": { + "description": "from is a list of sources which should be able to access the pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all sources (traffic not restricted by source). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the from list.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyPeer" + } + }, + "ports": { + "description": "ports is a list of ports which should be made accessible on the pods selected for this rule. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.NetworkPolicyPort" + } + } + } + }, + "io.k8s.api.networking.v1.NetworkPolicyPeer": { + "description": "NetworkPolicyPeer describes a peer to allow traffic to/from. Only certain combinations of fields are allowed", + "type": "object", + "properties": { + "ipBlock": { + "description": "ipBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.api.networking.v1.IPBlock" + }, + "namespaceSelector": { + "description": "namespaceSelector selects namespaces using cluster-scoped labels. This field follows standard label selector semantics; if present but empty, it selects all namespaces.\n\nIf podSelector is also set, then the NetworkPolicyPeer as a whole selects the pods matching podSelector in the namespaces selected by namespaceSelector. Otherwise it selects all pods in the namespaces selected by namespaceSelector.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "podSelector": { + "description": "podSelector is a label selector which selects pods. This field follows standard label selector semantics; if present but empty, it selects all pods.\n\nIf namespaceSelector is also set, then the NetworkPolicyPeer as a whole selects the pods matching podSelector in the Namespaces selected by NamespaceSelector. Otherwise it selects the pods matching podSelector in the policy's own namespace.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + } + } + }, + "io.k8s.api.networking.v1.NetworkPolicyPort": { + "description": "NetworkPolicyPort describes a port to allow traffic on", + "type": "object", + "properties": { + "endPort": { + "description": "endPort indicates that the range of ports from port to endPort if set, inclusive, should be allowed by the policy. This field cannot be defined if the port field is not defined or if the port field is defined as a named (string) port. The endPort must be equal or greater than port.", + "type": "integer", + "format": "int32" + }, + "port": { + "description": "port represents the port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers. If present, only traffic on the specified protocol AND port will be matched.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + }, + "protocol": { + "description": "protocol represents the protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.", + "type": "string" + } + } + }, + "io.k8s.api.policy.v1.PodDisruptionBudgetSpec": { + "description": "PodDisruptionBudgetSpec is a description of a PodDisruptionBudget.", + "type": "object", + "properties": { + "maxUnavailable": { + "description": "An eviction is allowed if at most \"maxUnavailable\" pods selected by \"selector\" are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. This is a mutually exclusive setting with \"minAvailable\".", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + }, + "minAvailable": { + "description": "An eviction is allowed if at least \"minAvailable\" pods selected by \"selector\" will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying \"100%\".", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.util.intstr.IntOrString" + }, + "selector": { + "description": "Label query over pods whose evictions are managed by the disruption budget. A null selector will match no pods, while an empty ({}) selector will select all pods within the namespace.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "unhealthyPodEvictionPolicy": { + "description": "UnhealthyPodEvictionPolicy defines the criteria for when unhealthy pods should be considered for eviction. Current implementation considers healthy pods, as pods that have status.conditions item with type=\"Ready\",status=\"True\".\n\nValid policies are IfHealthyBudget and AlwaysAllow. If no policy is specified, the default behavior will be used, which corresponds to the IfHealthyBudget policy.\n\nIfHealthyBudget policy means that running pods (status.phase=\"Running\"), but not yet healthy can be evicted only if the guarded application is not disrupted (status.currentHealthy is at least equal to status.desiredHealthy). Healthy pods will be subject to the PDB for eviction.\n\nAlwaysAllow policy means that all running pods (status.phase=\"Running\"), but not yet healthy are considered disrupted and can be evicted regardless of whether the criteria in a PDB is met. This means perspective running pods of a disrupted application might not get a chance to become healthy. Healthy pods will be subject to the PDB for eviction.\n\nAdditional policies may be added in the future. Clients making eviction decisions should disallow eviction of unhealthy pods if they encounter an unrecognized policy in this field.", + "type": "string" + } + } + }, + "io.k8s.apimachinery.pkg.api.resource.Quantity": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.FieldsV1": { + "description": "FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.\n\nEach key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:\u003cname\u003e', where \u003cname\u003e is the name of a field in a struct, or key in a map 'v:\u003cvalue\u003e', where \u003cvalue\u003e is the exact json formatted value of a list item 'i:\u003cindex\u003e', where \u003cindex\u003e is position of a item in a list 'k:\u003ckeys\u003e', where \u003ckeys\u003e is a map of a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.\n\nThe exact format is defined in sigs.k8s.io/structured-merge-diff", + "type": "object" + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector": { + "description": "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", + "type": "object", + "properties": { + "matchExpressions": { + "description": "matchExpressions is a list of label selector requirements. The requirements are ANDed.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelectorRequirement" + } + }, + "matchLabels": { + "description": "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelectorRequirement": { + "description": "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", + "type": "object", + "required": [ + "key", + "operator" + ], + "properties": { + "key": { + "description": "key is the label key that the selector applies to.", + "type": "string" + }, + "operator": { + "description": "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + "type": "string" + }, + "values": { + "description": "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.ManagedFieldsEntry": { + "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", + "type": "object", + "properties": { + "apiVersion": { + "description": "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", + "type": "string" + }, + "fieldsType": { + "description": "FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: \"FieldsV1\"", + "type": "string" + }, + "fieldsV1": { + "description": "FieldsV1 holds the first JSON version format as described in the \"FieldsV1\" type.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.FieldsV1" + }, + "manager": { + "description": "Manager is an identifier of the workflow managing these fields.", + "type": "string" + }, + "operation": { + "description": "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", + "type": "string" + }, + "subresource": { + "description": "Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.", + "type": "string" + }, + "time": { + "description": "Time is the timestamp of when the ManagedFields entry was added. The timestamp will also be updated if a field is added, the manager changes any of the owned fields value or removes a field. The timestamp does not update when a field is removed from the entry because another manager took it over.", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time" + } + } + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta": { + "description": "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", + "type": "object", + "properties": { + "annotations": { + "description": "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "creationTimestamp": { + "description": "CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n\nPopulated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time" + }, + "deletionGracePeriodSeconds": { + "description": "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", + "type": "integer", + "format": "int64" + }, + "deletionTimestamp": { + "description": "DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested.\n\nPopulated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Time" + }, + "finalizers": { + "description": "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order. Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.", + "type": "array", + "items": { + "type": "string" + } + }, + "generateName": { + "description": "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will return a 409.\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency", + "type": "string" + }, + "generation": { + "description": "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", + "type": "integer", + "format": "int64" + }, + "labels": { + "description": "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "managedFields": { + "description": "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ManagedFieldsEntry" + } + }, + "name": { + "description": "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names", + "type": "string" + }, + "namespace": { + "description": "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces", + "type": "string" + }, + "ownerReferences": { + "description": "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", + "type": "array", + "items": { + "$ref": "#/$defs/_definitions.json/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.OwnerReference" + } + }, + "resourceVersion": { + "description": "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency", + "type": "string" + }, + "selfLink": { + "description": "Deprecated: selfLink is a legacy read-only field that is no longer populated by the system.", + "type": "string" + }, + "uid": { + "description": "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + "type": "string" + } + } + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.OwnerReference": { + "description": "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", + "type": "object", + "required": [ + "apiVersion", + "kind", + "name", + "uid" + ], + "properties": { + "apiVersion": { + "description": "API version of the referent.", + "type": "string" + }, + "blockOwnerDeletion": { + "description": "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. See https://kubernetes.io/docs/concepts/architecture/garbage-collection/#foreground-deletion for how the garbage collector interacts with this field and enforces the foreground deletion. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", + "type": "boolean" + }, + "controller": { + "description": "If true, this reference points to the managing controller.", + "type": "boolean" + }, + "kind": { + "description": "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "name": { + "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names", + "type": "string" + }, + "uid": { + "description": "UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + "type": "string" + } + } + }, + "io.k8s.apimachinery.pkg.apis.meta.v1.Time": { + "description": "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + "type": "string", + "format": "date-time" + }, + "io.k8s.apimachinery.pkg.util.intstr.IntOrString": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + } + } + } + } +} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.yaml new file mode 100644 index 00000000..543fe0e5 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/node-feature-discovery/values.yaml @@ -0,0 +1,1261 @@ +image: + # -- NFD image repository + # @section -- General + repository: gcr.io/k8s-staging-nfd/node-feature-discovery + # @schema enum: [Always, IfNotPresent, Never] + # -- Image pull policy + # @section -- General + pullPolicy: Always + # @schema type: [string, null] + # -- NFD image tag. If not specified Chart.AppVersion will be used. + # @section -- General + tag: +# @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/imagePullSecrets +# -- Image pull secrets. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod). +# @section -- General +imagePullSecrets: [] + +# -- Override the name of the chart +# @section -- General +nameOverride: "" +# -- Override a default fully qualified app name +# @section -- General +fullnameOverride: "" +# -- Override the namespace to install the chart into. By default, the namespace is determined by Helm. +# @section -- General +namespaceOverride: "" + +# @schema patternProperties: {"^[a-zA-Z]$": {type: string}} +# -- [Feature gates](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/feature-gates) to enable/disable specific features. +# @section -- General +featureGates: {} + +# -- Only checksum config values (not the full rendered ConfigMap template) for the rollout annotation. +# When enabled, pods will only restart when config values actually change, not on unrelated template changes. +# @section -- General +checksumValuesOnly: false + +# -- The name of the PriorityClass to be used for the NFD pods. +# @section -- General +priorityClassName: "" + +# -- Enable/disable the Helm post-delete hook. +# @section -- General +postDeleteCleanup: true + +# +# Global configuration for NFD used as a sub-chart +# +global: + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/imagePullSecrets + # -- If `imagePullSecrets` is specified, it takes precedence over `global.imagePullSecrets`. [More info](https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod). + # @section -- Global + imagePullSecrets: [] + +# +# NFD-Master configuration +# +master: + # -- Specifies whether nfd-master should be deployed + # @section -- NFD-Master + enable: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/args + # -- Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/master-commandline-reference) to pass to nfd-master. + # @section -- NFD-Master + extraArgs: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/env + # -- Additional environment variables to set in the nfd-master container. + # @section -- NFD-Master + extraEnvs: [] + # -- Run the container in the host's network namespace. + # @section -- NFD-Master + hostNetwork: false + # @schema type: [boolean, null] + # -- (bool) Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. + # @section -- NFD-Master + hostUsers: ~ # @schema type: [boolean, null] + # @enum: [Default, ClusterFirst, ClusterFirstWithHostNet, None] + # -- NFD master pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy). + # @section -- NFD-Master + dnsPolicy: ClusterFirstWithHostNet + # @schema type: [object, null] + # -- NFD master [configuration](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/master-configuration-reference). + # @section -- NFD-Master + config: ### + # noPublish: false + # extraLabelNs: ["added.ns.io","added.kubernets.io"] + # denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] + # enableTaints: false + # informerPageSize: 200 + # labelWhiteList: "foo" + # resyncPeriod: "2h" + # restrictions: + # disableLabels: true + # disableTaints: true + # disableExtendedResources: true + # disableAnnotations: true + # allowOverwrite: false + # denyNodeFeatureLabels: true + # nodeFeatureNamespaceSelector: + # matchLabels: + # kubernetes.io/metadata.name: "node-feature-discovery" + # matchExpressions: + # - key: "kubernetes.io/metadata.name" + # operator: "In" + # values: + # - "node-feature-discovery" + # klog: + # addDirHeader: false + # alsologtostderr: false + # logBacktraceAt: + # logtostderr: true + # skipHeaders: false + # stderrthreshold: 2 + # v: 0 + # vmodule: + ## NOTE: the following options are not dynamically run-time configurable + ## and require a nfd-master restart to take effect after being changed + # logDir: + # logFile: + # logFileMaxSize: 1800 + # skipLogHeaders: false + # leaderElection: + # leaseDuration: 15s + # # this value has to be lower than leaseDuration and greater than retryPeriod*1.2 + # renewDeadline: 10s + # # this value has to be greater than 0 + # retryPeriod: 2s + # nfdApiParallelism: 10 + ### + + # @schema type: integer; minimum: 1; maximum: 65535 + # -- Port on which to serve http for metrics and healthz endpoints. + # @section -- NFD-Master + port: 8080 + # @schema type: [string, null] + # -- Instance name. Used to separate annotation namespaces for multiple parallel deployments. + # @section -- NFD-Master + instance: + # @schema type: [string, null] + # -- (int) NFD API controller resync period. Time duration string (e.g. "5m", "1h", "2h45m"). + # @section -- NFD-Master + resyncPeriod: + # @schema item: string + # -- Label namespaces to deny. Labels with these prefixes will not be published to the nodes. + # @section -- NFD-Master + denyLabelNs: [] + # @schema item: string + # -- Additional label namespaces to publish. Labels with these prefixes are allowed even if otherwise denied by `master.denyLabelNs`. + # @section -- NFD-Master + extraLabelNs: [] + # -- Enable node tainting. + # @section -- NFD-Master + enableTaints: false + # @schema type: [integer, null]; minimum: 1; maximum: 100 + # -- (int) The maximum number of concurrent node updates. + # @section -- NFD-Master + nfdApiParallelism: + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-master Deployment. + # @section -- NFD-Master + deploymentAnnotations: {} + # -- Number of the desired replicas for the nfd-master Deployment. + # @section -- NFD-Master + replicaCount: 1 + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext + # -- [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-master pods. + # @section -- NFD-Master + podSecurityContext: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers + # -- [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-master pods. + # @section -- NFD-Master + initContainers: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext + # -- [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-master container. + # @section -- NFD-Master + securityContext: + # @schema hidden + allowPrivilegeEscalation: false + # @schema hidden + capabilities: + drop: [ "ALL" ] + # @schema hidden + readOnlyRootFilesystem: true + # @schema hidden + runAsNonRoot: true + serviceAccount: + # -- Specifies whether a service account should be created. + # @section -- NFD-Master + create: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account + # @section -- NFD-Master + annotations: {} + # @schema type: [string, null] + # -- The name of the service account to use. If not set and create is true, a name is generated using the fullname template + # @section -- NFD-Master + name: + # @schema type: [integer, null] + # -- (int) Specifies the number of old ReplicaSets for the Deployment to retain. [revisionHistoryLimit](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#revision-history-limit) + # @section -- NFD-Master + revisionHistoryLimit: + rbac: + # -- Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-master. + # @section -- NFD-Master + create: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements + resources: + # @schema hidden + # -- Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-master container. + # @section -- NFD-Master + limits: + memory: 4Gi + # @schema hidden + # -- Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-master container. + # @section -- NFD-Master + requests: + cpu: 100m + # You may want to use the same value for `requests.memory` and `limits.memory`. The “requests” value affects scheduling to accommodate pods on nodes. + # If there is a large difference between “requests” and “limits” and nodes experience memory pressure, the kernel may invoke + # the OOM Killer, even if the memory does not exceed the “limits” threshold. This can cause unexpected pod evictions. Memory + # cannot be compressed and once allocated to a pod, it can only be reclaimed by killing the pod. + # Natan Yellin 22/09/2022 https://home.robusta.dev/blog/kubernetes-memory-limit + memory: 128Mi + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector + # -- [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-master pods. + # @section -- NFD-Master + nodeSelector: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations + # -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-master pods. + # @section -- NFD-Master + tolerations: + - # @schema hidden + key: "node-role.kubernetes.io/control-plane" + # @schema hidden + operator: "Equal" + # @schema hidden + value: "" + # @schema hidden + effect: "NoSchedule" + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.policy.v1.PodDisruptionBudgetSpec + podDisruptionBudget: + # -- Configure PodDisruptionBudget for the nfd-master Deployment. + # @section -- NFD-Master + enable: false + # @schema hidden + # -- Minimum number (or percentage) of pods that must be available after the eviction. + # @section -- NFD-Master + minAvailable: 1 + # @schema hidden + # -- Policy to evict unhealthy pods when a PodDisruptionBudget is defined. + # @section -- NFD-Master + unhealthyPodEvictionPolicy: AlwaysAllow + # @section -- NFD-Master + networkPolicy: + # -- Should a networkPolicy be deployed for the nfd-master pods + # @section -- NFD-Master + enabled: false + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule + # -- [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-master pods. + # The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp + # @section -- NFD-Master + egress: + - ports: # @schema hidden + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + - port: 6443 + protocol: TCP + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule + # -- [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-master pods. + # @section -- NFD-Master + ingress: + - ports: # @schema hidden + - protocol: TCP + port: http + + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-master pods. + # @section -- NFD-Master + annotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels + # -- [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-master pods. + # @section -- NFD-Master + labels: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity + # -- [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-master pods. + # @section -- NFD-Master + affinity: + # @schema hidden + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [""] + # -- Startup probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-startup-probes). + # @section -- NFD-Master + # @default -- - + startupProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Master + initialDelaySeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Master + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Master + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of consecutive failures for the probe before considering the pod as not ready. + # @section -- NFD-Master + failureThreshold: 30 + # -- Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). + # @section -- NFD-Master + # @default -- - + livenessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Master + initialDelaySeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Master + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Master + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-Master + failureThreshold: + # -- Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). + # @section -- NFD-Master + # @default -- - + readinessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Master + initialDelaySeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Master + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Master + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-Master + successThreshold: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of consecutive failures for the probe before considering the pod as not ready. + # @section -- NFD-Master + failureThreshold: 10 + +# +# NFD-Worker configuration +# +worker: + # -- Specifies whether nfd-worker should be deployed + # @section -- NFD-Worker + enable: true + # @schema uniqueItems:true;item:string;itemEnum:["node","pod","ds"] + # -- Objects used as owner references for NodeFeature objects. Valid values are `node`, `pod`, and `ds`. This value is passed through `-owner-refs` and takes precedence over `worker.config.core.ownerRefs`. + # @section -- NFD-Worker + ownerRefs: + - pod + - ds + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/args + # -- Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/worker-commandline-reference) to pass to nfd-worker. + # @section -- NFD-Worker + extraArgs: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/env + # -- Additional environment variables to set in the nfd-worker container. + # @section -- NFD-Worker + extraEnvs: [] + # -- Run the container in the host's network namespace. + # @section -- NFD-Worker + hostNetwork: false + # @schema type: [boolean, null] + # -- (bool) Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. + # @section -- NFD-Worker + hostUsers: ~ # @schema type: [boolean, null] + # @enum: [Default, ClusterFirst, ClusterFirstWithHostNet, None] + # -- NFD worker pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy). + # @section -- NFD-Worker + dnsPolicy: ClusterFirstWithHostNet + # @schema type: [object, null] + # -- NFD worker [configuration](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/worker-configuration-reference). + # @section -- NFD-Worker + config: ### + #core: + # labelWhiteList: + # noPublish: false + # noOwnerRefs: false + # # Select from node, pod and ds. Use [] to disable owner references. + # ownerRefs: [pod, ds] + # sleepInterval: 60s + # featureSources: [all] + # labelSources: [all] + # # Discover these features but omit them from the published NodeFeature object + # # to reduce its size. Exact key or '*' prefix (e.g. pci.device, pci.*). + # # Only for features no NodeFeatureRule/NodeFeatureGroup consumes. + # noPublishFeatures: [] + # klog: + # addDirHeader: false + # alsologtostderr: false + # logBacktraceAt: + # logtostderr: true + # skipHeaders: false + # stderrthreshold: 2 + # v: 0 + # vmodule: + ## NOTE: the following options are not dynamically run-time configurable + ## and require a nfd-worker restart to take effect after being changed + # logDir: + # logFile: + # logFileMaxSize: 1800 + # skipLogHeaders: false + #sources: + # cpu: + # cpuid: + ## NOTE: whitelist has priority over blacklist + # attributeBlacklist: + # - "AVX10" + # - "BMI1" + # - "BMI2" + # - "CLMUL" + # - "CMOV" + # - "CX16" + # - "ERMS" + # - "F16C" + # - "HTT" + # - "LZCNT" + # - "MMX" + # - "MMXEXT" + # - "NX" + # - "POPCNT" + # - "RDRAND" + # - "RDSEED" + # - "RDTSCP" + # - "SGX" + # - "SSE" + # - "SSE2" + # - "SSE3" + # - "SSE4" + # - "SSE42" + # - "SSSE3" + # - "TDX_GUEST" + # attributeWhitelist: + # kernel: + # kconfigFile: "/path/to/kconfig" + # configOpts: + # - "NO_HZ" + # - "X86" + # - "DMI" + # pci: + # deviceClassWhitelist: + # - "0200" + # - "03" + # - "12" + # deviceLabelFields: + # - "class" + # - "vendor" + # - "device" + # - "subsystem_vendor" + # - "subsystem_device" + # usb: + # deviceClassWhitelist: + # - "0e" + # - "ef" + # - "fe" + # - "ff" + # deviceLabelFields: + # - "class" + # - "vendor" + # - "device" + # custom: + # # The following feature demonstrates the capabilities of the matchFeatures + # - name: "my custom rule" + # labels: + # "vendor.io/my-ng-feature": "true" + # # matchFeatures implements a logical AND over all matcher terms in the + # # list (i.e. all of the terms, or per-feature matchers, must match) + # matchFeatures: + # - feature: cpu.cpuid + # matchExpressions: + # AVX512F: {op: Exists} + # - feature: cpu.cstate + # matchExpressions: + # enabled: {op: IsTrue} + # - feature: cpu.pstate + # matchExpressions: + # no_turbo: {op: IsFalse} + # scaling_governor: {op: In, value: ["performance"]} + # - feature: cpu.rdt + # matchExpressions: + # RDTL3CA: {op: Exists} + # - feature: cpu.sst + # matchExpressions: + # bf.enabled: {op: IsTrue} + # - feature: cpu.topology + # matchExpressions: + # hardware_multithreading: {op: IsFalse} + # + # - feature: kernel.config + # matchExpressions: + # X86: {op: Exists} + # LSM: {op: InRegexp, value: ["apparmor"]} + # - feature: kernel.loadedmodule + # matchExpressions: + # e1000e: {op: Exists} + # - feature: kernel.selinux + # matchExpressions: + # enabled: {op: IsFalse} + # - feature: kernel.kvm + # matchExpressions: + # enabled: {op: IsFalse} + # - feature: kernel.version + # matchExpressions: + # major: {op: In, value: ["5"]} + # minor: {op: Gt, value: ["10"]} + # + # - feature: storage.block + # matchExpressions: + # rotational: {op: In, value: ["0"]} + # dax: {op: In, value: ["0"]} + # + # - feature: network.device + # matchExpressions: + # operstate: {op: In, value: ["up"]} + # speed: {op: Gt, value: ["100"]} + # + # - feature: memory.numa + # matchExpressions: + # node_count: {op: Gt, value: ["2"]} + # - feature: memory.nv + # matchExpressions: + # devtype: {op: In, value: ["nd_dax"]} + # mode: {op: In, value: ["memory"]} + # + # - feature: system.osrelease + # matchExpressions: + # ID: {op: In, value: ["fedora", "centos"]} + # - feature: system.name + # matchExpressions: + # nodename: {op: InRegexp, value: ["^worker-X"]} + # + # - feature: local.label + # matchExpressions: + # custom-feature-knob: {op: Gt, value: ["100"]} + # + # # The following feature demonstrates the capabilities of the matchAny + # - name: "my matchAny rule" + # labels: + # "vendor.io/my-ng-feature-2": "my-value" + # # matchAny implements a logical IF over all elements (sub-matchers) in + # # the list (i.e. at least one feature matcher must match) + # matchAny: + # - matchFeatures: + # - feature: kernel.loadedmodule + # matchExpressions: + # driver-module-X: {op: Exists} + # - feature: pci.device + # matchExpressions: + # vendor: {op: In, value: ["8086"]} + # class: {op: In, value: ["0200"]} + # - matchFeatures: + # - feature: kernel.loadedmodule + # matchExpressions: + # driver-module-Y: {op: Exists} + # - feature: usb.device + # matchExpressions: + # vendor: {op: In, value: ["8086"]} + # class: {op: In, value: ["02"]} + # + # - name: "avx wildcard rule" + # labels: + # "my-avx-feature": "true" + # matchFeatures: + # - feature: cpu.cpuid + # matchName: {op: InRegexp, value: ["^AVX512"]} + # + # # The following features demonstreate label templating capabilities + # - name: "my template rule" + # labelsTemplate: | + # {{ range .system.osrelease }}vendor.io/my-system-feature.{{ .Name }}={{ .Value }} + # {{ end }} + # matchFeatures: + # - feature: system.osrelease + # matchExpressions: + # ID: {op: InRegexp, value: ["^open.*"]} + # VERSION_ID.major: {op: In, value: ["13", "15"]} + # + # - name: "my template rule 2" + # labelsTemplate: | + # {{ range .pci.device }}vendor.io/my-pci-device.{{ .class }}-{{ .device }}=with-cpuid + # {{ end }} + # matchFeatures: + # - feature: pci.device + # matchExpressions: + # class: {op: InRegexp, value: ["^06"]} + # vendor: ["8086"] + # - feature: cpu.cpuid + # matchExpressions: + # AVX: {op: Exists} + # + # # The following examples demonstrate vars field and back-referencing + # # previous labels and vars + # - name: "my dummy kernel rule" + # labels: + # "vendor.io/my.kernel.feature": "true" + # matchFeatures: + # - feature: kernel.version + # matchExpressions: + # major: {op: Gt, value: ["2"]} + # + # - name: "my dummy rule with no labels" + # vars: + # "my.dummy.var": "1" + # matchFeatures: + # - feature: cpu.cpuid + # matchExpressions: {} + # + # - name: "my rule using backrefs" + # labels: + # "vendor.io/my.backref.feature": "true" + # matchFeatures: + # - feature: rule.matched + # matchExpressions: + # vendor.io/my.kernel.feature: {op: IsTrue} + # my.dummy.var: {op: Gt, value: ["0"]} + # + # - name: "kconfig template rule" + # labelsTemplate: | + # {{ range .kernel.config }}kconfig-{{ .Name }}={{ .Value }} + # {{ end }} + # matchFeatures: + # - feature: kernel.config + # matchName: {op: In, value: ["SWAP", "X86", "ARM"]} +### + + # @schema type: integer; minimum: 1; maximum: 65535 + # -- Port on which to serve http for metrics and healthz endpoints. + # @section -- NFD-Worker + port: 8080 + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-worker DaemonSet. + # @section -- NFD-Worker + daemonsetAnnotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext + # -- [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-worker pods. + # @section -- NFD-Worker + podSecurityContext: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers + # -- [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-worker pods. + # @section -- NFD-Worker + initContainers: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext + # -- [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-worker container. + # @section -- NFD-Worker + securityContext: + # @schema hidden + allowPrivilegeEscalation: false + # @schema hidden + capabilities: + drop: [ "ALL" ] + # @schema hidden + readOnlyRootFilesystem: true + # @schema hidden + runAsNonRoot: true + + # -- Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). + # @section -- NFD-Worker + # @default -- - + livenessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Worker + initialDelaySeconds: 10 + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Worker + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Worker + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-Worker + failureThreshold: + # -- Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). + # @section -- NFD-Worker + # @default -- - + readinessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Worker + initialDelaySeconds: 5 + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Worker + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Worker + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-Worker + successThreshold: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of consecutive failures for the probe before considering the pod as not ready. + # @section -- NFD-Worker + failureThreshold: 10 + + serviceAccount: + # -- Specifies whether a service account should be created. + # @section -- NFD-Worker + create: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account + # @section -- NFD-Worker + annotations: {} + # @schema type: [string, null] + # -- Name of the service account to use. If not set and create is true, a name is generated using the fullname template + # @section -- NFD-Worker + name: + # @schema type: [integer, null] + # -- (int) Specifies the number of old history for the DaemonSet to retain to allow rollback. + # @section -- NFD-Worker + revisionHistoryLimit: + rbac: + # -- Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-worker. + # @section -- NFD-Worker + create: true + + # Allow users to mount the hostPath /usr/src, useful for RHCOS on s390x + # Does not work on systems without /usr/src AND a read-only /usr, such as Talos + # -- Mount host path /user/src inside the container. Does not work on systems without /usr/src AND a read-only /usr. + # @section -- NFD-Worker + mountUsrSrc: false + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements + resources: + # @schema hidden + # -- Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-worker container. + # @section -- NFD-Worker + limits: + memory: 512Mi + # @schema hidden + # -- Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-worker container. + # @section -- NFD-Worker + requests: + cpu: 5m + memory: 64Mi + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector + # -- [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-worker pods. + # @section -- NFD-Worker + nodeSelector: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations + # -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-worker pods. + # @section -- NFD-Worker + tolerations: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-worker pods. + # @section -- NFD-Worker + annotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels + # -- [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-worker pods. + # @section -- NFD-Worker + labels: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity + # -- [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-worker pods. + # @section -- NFD-Worker + affinity: {} + # @schema type: [string, null] + # -- The name of the PriorityClass to be used for the nfd-worker pods. + # @section -- NFD-Worker + priorityClassName: + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.apps.v1.DaemonSetUpdateStrategy + # -- Update strategy for the nfd-worker DaemonSet. Defaults to a rolling + # update with `maxUnavailable: "10%"` so upgrades complete in a bounded + # number of waves on clusters of any size (the Kubernetes default of + # `maxUnavailable: 1` rolls one node at a time and makes Helm wait and + # Flux HelmRelease timeouts likely on large clusters). nfd-worker is + # stateless and node labels persist while a worker pod restarts, so a + # faster roll is safe. Set `maxUnavailable: 1` to restore the Kubernetes + # default. To use `type: OnDelete`, also set `rollingUpdate: null` + # (Helm deep-merges maps). + # [More info](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set) + # @section -- NFD-Worker + updateStrategy: + rollingUpdate: + # @schema type: [string, integer] + maxUnavailable: "10%" + # @section -- NFD-Worker + networkPolicy: + # -- Should a networkPolicy be deployed for the nfd-worker pods + # @section -- NFD-Worker + enabled: false + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule + # -- [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-worker pods. + # The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp + # @section -- NFD-Worker + egress: + - ports: # @schema hidden + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + - port: 6443 + protocol: TCP + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule + # -- [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-worker pods. + # @section -- NFD-Worker + ingress: + - ports: # @schema hidden + - protocol: TCP + port: http + +# +# NFD-Topology-Updater configuration +# +topologyUpdater: + # @schema type: [object, null] + # -- Configuration for the topology updater. See the [configuration reference](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/topology-updater-configuration-reference) for details. + # @section -- NFD-Topology-Updater + config: ### + ## key = node name, value = list of resources to be excluded. + ## use * to exclude from all nodes. + ## an example for how the exclude list should looks like + #excludeList: + # node1: [cpu] + # node2: [memory, example/deviceA] + # *: [hugepages-2Mi] +### + + # -- Specifies whether nfd-topology-updater should be deployed. + # @section -- NFD-Topology-Updater + enable: false + # -- Create the NodeResourceTopology CRD. This MUST be set to true when + # 'enable' is true, unless the CRD is installed separately (e.g., by another + # Helm release or external tool). If the CRD is missing, the topology-updater + # pods will fail with "NodeResourceTopology CRD is not installed" error. + # @section -- NFD-Topology-Updater + createCRDs: false + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/args + # -- Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/topology-updater-commandline-reference) to pass to nfd-topology-updater. + # @section -- NFD-Topology-Updater + extraArgs: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/env + # -- Additional environment variables to set in the nfd-topology-updater container. + # @section -- NFD-Topology-Updater + extraEnvs: [] + # -- Run the container in the host's network namespace. + # @section -- NFD-Topology-Updater + hostNetwork: false + # @schema type: [boolean, null] + # -- (bool) Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. + # @section -- NFD-Topology-Updater + hostUsers: ~ # @schema type: [boolean, null] + # @enum: [Default, ClusterFirst, ClusterFirstWithHostNet, None] + # -- NFD topology updater pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy). + # @section -- NFD-Topology-Updater + dnsPolicy: ClusterFirstWithHostNet + serviceAccount: + # -- Specifies whether a service account should be created. + # @section -- NFD-Topology-Updater + create: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account. + # @section -- NFD-Topology-Updater + annotations: {} + # @schema type: [string, null] + # -- Name or the service account to use. If not set and create is true, a name is generated using the fullname template. + # @section -- NFD-Topology-Updater + name: + + # @schema type: [integer, null] + # -- (int) Specifies the number of old history for the DaemonSet to retain to allow rollback. + # @section -- NFD-Topology-Updater + revisionHistoryLimit: + rbac: + # -- Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-topology-updater. + # @section -- NFD-Topology-Updater + create: true + # @schema type: integer; minimum: 1; maximum: 65535 + # -- Port on which to serve http for metrics and healthz endpoints. + # @section -- NFD-Topology-Updater + port: 8080 + # @schema type: [string, null] + # -- Host path for the kubelet config file. + # @section -- NFD-Topology-Updater + kubeletConfigPath: + # @schema type: [string, null] + # -- Host path for the kubelet socket for podresources endpoint. + # @section -- NFD-Topology-Updater + kubeletPodResourcesSockPath: + # -- Time to sleep between CR updates. Non-positive value implies no CR update. + # @section -- NFD-Topology-Updater + updateInterval: 60s + # -- Namespace to watch pods, `*` for all namespaces. + # @section -- NFD-Topology-Updater + watchNamespace: "*" + # -- The kubelet state directory path for watching state and checkpoint files. Empty value disables kubelet state tracking. + # @section -- NFD-Topology-Updater + kubeletStateDir: /var/lib/kubelet + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext + # -- [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + podSecurityContext: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers + # -- [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + initContainers: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext + # -- [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) of the nfd-topology-updater container. + # @section -- NFD-Topology-Updater + securityContext: + # @schema hidden + allowPrivilegeEscalation: false + # @schema hidden + capabilities: + drop: [ "ALL" ] + # @schema hidden + readOnlyRootFilesystem: true + # @schema hidden + runAsUser: 0 + + # -- Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). + # @section -- NFD-Topology-Updater + # @default -- - + livenessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Topology-Updater + initialDelaySeconds: 10 + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Topology-Updater + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Topology-Updater + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-Topology-Updater + failureThreshold: + # -- Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). + # @section -- NFD-Topology-Updater + # @default -- - + readinessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-Topology-Updater + initialDelaySeconds: 5 + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-Topology-Updater + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-Topology-Updater + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-Topology-Updater + successThreshold: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of consecutive failures for the probe before considering the pod as not ready. + # @section -- NFD-Topology-Updater + failureThreshold: 10 + + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements + resources: + # @schema hidden + # -- Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-topology-updater container. + # @section -- NFD-Topology-Updater + limits: + memory: 60Mi + # @schema hidden + # -- Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-topology-updater container. + # @section -- NFD-Topology-Updater + requests: + cpu: 50m + memory: 40Mi + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector + # -- [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + nodeSelector: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations + # -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + tolerations: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + annotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels + # -- [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + labels: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-topology-updater DaemonSet. + # @section -- NFD-Topology-Updater + daemonsetAnnotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity + # -- [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-topology-updater pods. + # @section -- NFD-Topology-Updater + affinity: {} + # -- Enables compute and report of pod fingerprint in NRT objects. + # @section -- NFD-Topology-Updater + podSetFingerprint: true + # @section -- NFD-Topology-Updater + networkPolicy: + # -- Should a networkPolicy be deployed for the nfd-topology pods + # @section -- NFD-Topology-Updater + enabled: false + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule + # -- [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-topology pods. + # The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp + # @section -- NFD-Topology-Updater + egress: + - ports: # @schema hidden + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + - port: 6443 + protocol: TCP + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule + # -- [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-topology pods. + # @section -- NFD-Topology-Updater + ingress: + - ports: # @schema hidden + - protocol: TCP + port: http + +# +# NFD-GC configuration +# +gc: + # -- Specifies whether nfd-gc should be deployed. + # @section -- NFD-GC + enable: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/args + # -- Additional [command line arguments](https://kubernetes-sigs.github.io/node-feature-discovery/master/reference/gc-commandline-reference) to pass to nfd-gc. + # @section -- NFD-GC + extraArgs: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Container/properties/env + # -- Additional environment variables to set in the nfd-gc container. + # @section -- NFD-GC + extraEnvs: [] + # -- Run the container in the host's network namespace. + # @section -- NFD-GC + hostNetwork: false + # @schema type: [boolean, null] + # -- (bool) Run the container with host user ids. NOTE: if hostNetwork is true, hostUsers should be true. + # @section -- NFD-GC + hostUsers: ~ # @schema type: [boolean, null] + # -- The number of desired replicas for the nfd-gc Deployment. + # @section -- NFD-GC + replicaCount: 1 + # @enum: [Default, ClusterFirst, ClusterFirstWithHostNet, None] + # -- NFD gc pod [dnsPolicy](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-dns-policy). + # @section -- NFD-GC + dnsPolicy: ClusterFirstWithHostNet + serviceAccount: + # -- Specifies whether a service account should be created. + # @section -- NFD-GC + create: true + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the service account. + # @section -- NFD-GC + annotations: {} + # @schema type: [string, null] + # -- Name of the service account to use. If not set and create is true, a name is generated using the fullname template. + # @section -- NFD-GC + name: + rbac: + # -- Create [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) configuration for nfd-gc. + # @section -- NFD-GC + create: true + # -- Time between periodic garbage collector runs. + # @section -- NFD-GC + interval: 1h + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSecurityContext + # -- [Pod SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) of the nfd-gc pods. + # @section -- NFD-GC + podSecurityContext: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/initContainers + # -- [Init containers](https://kubernetes.io/docs/concepts/workloads/pods/init-containers/) to add to the nfd-gc pods. + # @section -- NFD-GC + initContainers: [] + + # -- Liveness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-liveness-probes). + # @section -- NFD-GC + # @default -- - + livenessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-GC + initialDelaySeconds: 10 + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-GC + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-GC + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-GC + failureThreshold: + # -- Readiness probe configuration. [More information](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-readiness-probes). + # @section -- NFD-GC + # @default -- - + readinessProbe: + # @schema type: [integer, null] + # -- (int) The number of seconds after the container has started before probe is initiated. + # @section -- NFD-GC + initialDelaySeconds: 5 + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of seconds after which the probe times out. + # @section -- NFD-GC + timeoutSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) How often (in seconds) to perform the probe. + # @section -- NFD-GC + periodSeconds: + # @schema type: [integer, null]; minimum: 1 + # -- (int) Minimum consecutive successes for the probe before considering the pod as ready. + # @section -- NFD-GC + successThreshold: + # @schema type: [integer, null]; minimum: 1 + # -- (int) The number of consecutive failures for the probe before considering the pod as not ready. + # @section -- NFD-GC + failureThreshold: + + resources: + # @schema hidden + # -- Resource [limits](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-gc container. + # @section -- NFD-GC + limits: + memory: 1Gi + # @schema hidden + # -- Resource [requests](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#requests-and-limits) for the nfd-gc container. + # @section -- NFD-GC + requests: + cpu: 10m + memory: 128Mi + # @schema type: integer; minimum: 1; maximum: 65535 + # -- Port on which to serve http for metrics and healthz endpoints. + # @section -- NFD-GC + port: 8080 + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/nodeSelector + # -- [Node selector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for the nfd-gc pods. + # @section -- NFD-GC + nodeSelector: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.PodSpec/properties/tolerations + # -- [Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for the nfd-gc pods. + # @section -- NFD-GC + tolerations: [] + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-gc pods. + # @section -- NFD-GC + annotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/labels + # -- [Labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to add to the nfd-gc pods. + # @section -- NFD-GC + labels: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta/properties/annotations + # -- [Annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations) to add to the nfd-gc Deployment. + # @section -- NFD-GC + deploymentAnnotations: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity + # -- [Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the nfd-gc pods. + # @section -- NFD-GC + affinity: {} + # @schema $ref: $k8s/_definitions.json#/definitions/io.k8s.api.policy.v1.PodDisruptionBudgetSpec + podDisruptionBudget: + # -- Configure PodDisruptionBudget for the nfd-master Deployment. + # @section -- NFD-GC + enable: false + # @schema hidden + # -- Minimum number (or percentage) of pods that must be available after the eviction. + # @section -- NFD-GC + minAvailable: 1 + # @schema hidden + # -- Policy to evict unhealthy pods when a PodDisruptionBudget is defined. + # @section -- NFD-GC + unhealthyPodEvictionPolicy: AlwaysAllow + # @schema type: [integer, null] + # -- (int) Specifies the number of old history for the Deployment to retain to allow rollback. + # @section -- NFD-GC + revisionHistoryLimit: + # @section -- NFD-GC + networkPolicy: + # -- Should a networkPolicy be deployed for the nfd-gc pods + # @section -- NFD-GC + enabled: false + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyEgressRule + # -- [Egress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-gc pods. + # The minimum egress ports required to function are: DNS (53/udp, 53/tcp, API server (80/tcp, 443/tcp, 6443/tcp). NOTE: OKD and Openshift use 6443/tcp + # @section -- NFD-GC + egress: + - ports: # @schema hidden + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP + - port: 53 + protocol: TCP + - port: 53 + protocol: UDP + - port: 6443 + protocol: TCP + # @schema itemRef: $k8s/_definitions.json#/definitions/io.k8s.api.networking.v1.NetworkPolicyIngressRule + # -- [Ingress](https://kubernetes.io/docs/concepts/services-networking/network-policies/#network-traffic-filtering) for the nfd-gc pods. + # @section -- NFD-GC + ingress: + - ports: # @schema hidden + - protocol: TCP + port: http + +# +# Prometheus configuration +# +prometheus: + # -- Create PodMonitor object for enabling metrics collection by Prometheus Operator. + # @section -- Prometheus + enable: false + # -- Interval at which metrics are scraped. + # @section -- Prometheus + scrapeInterval: 10s + # -- Labels to add to the PodMonitor object. + # @section -- Prometheus + labels: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/.helmignore b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/Chart.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/Chart.yaml new file mode 100644 index 00000000..e1751b88 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: sriov-network-operator +version: 0.1.0 +kubeVersion: '>= 1.32.0-0' +appVersion: 1.2.0 +description: SR-IOV network operator configures and manages SR-IOV networks in the kubernetes cluster +type: application +keywords: + - sriov +home: https://github.com/k8snetworkplumbingwg/sriov-network-operator +sources: + - https://github.com/k8snetworkplumbingwg/sriov-network-operator diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/README.md b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/README.md new file mode 100644 index 00000000..c17e4082 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/README.md @@ -0,0 +1,170 @@ +# SR-IOV Network Operator Helm Chart + +SR-IOV Network Operator Helm Chart provides an easy way to install, configure and manage +the lifecycle of SR-IOV network operator. + +## SR-IOV Network Operator +SR-IOV Network Operator leverages [Kubernetes CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) +and [Operator SDK](https://github.com/operator-framework/operator-sdk) to configure and manage SR-IOV networks in a Kubernetes cluster. + +SR-IOV Network Operator features: +- Initialize the supported SR-IOV NIC types on selected nodes. +- Provision/upgrade SR-IOV device plugin executable on selected node. +- Provision/upgrade SR-IOV CNI plugin executable on selected nodes. +- Manage configuration of SR-IOV device plugin on host. +- Generate net-att-def CRs for SR-IOV CNI plugin +- Supports operation in a virtualized Kubernetes deployment + - Discovers VFs attached to the Virtual Machine (VM) + - Does not require attached of associated PFs + - VFs can be associated to SriovNetworks by selecting the appropriate PciAddress as the RootDevice in the SriovNetworkNodePolicy + +## QuickStart + +### Prerequisites + +- Kubernetes v1.17+ +- Helm v3 + +### Install Helm + +Helm provides an install script to copy helm binary to your system: +``` +$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +$ chmod 500 get_helm.sh +$ ./get_helm.sh +``` + +For additional information and methods for installing Helm, refer to the official [helm website](https://helm.sh/) + +### Deploy SR-IOV Network Operator + +#### Deploy from OCI repo + +``` +$ helm install -n sriov-network-operator --create-namespace --version 1.5.0 --set sriovOperatorConfig.deploy=true sriov-network-operator oci://ghcr.io/k8snetworkplumbingwg/sriov-network-operator-chart +``` + +#### Deploy from project sources + +``` +# Clone project +$ git clone https://github.com/k8snetworkplumbingwg/sriov-network-operator.git ; cd sriov-network-operator + +# Install Operator +$ helm install -n sriov-network-operator --create-namespace --wait --set sriovOperatorConfig.deploy=true sriov-network-operator ./deployment/sriov-network-operator-chart + +# View deployed resources +$ kubectl -n sriov-network-operator get pods +``` + +In the case that [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) is enabled, the sriov network operator namespace will require a security level of 'privileged' +``` +$ kubectl label ns sriov-network-operator pod-security.kubernetes.io/enforce=privileged +``` + +## Chart parameters + +In order to tailor the deployment of the network operator to your cluster needs +We have introduced the following Chart parameters. + +| Name | Type | Default | description | +| ---- |------|---------|-------------| +| `imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling any of the SR-IOV Network Operator image | +| `supportedExtraNICs` | list | `[]` | An optional list of whitelisted NICs | + +### Operator parameters + +| Name | Type | Default | description | +| ---- | ---- | ------- | ----------- | +| `operator.tolerations` | list | `[{"key":"node-role.kubernetes.io/master","operator":"Exists","effect":"NoSchedule"},{"key":"node-role.kubernetes.io/control-plane","operator":"Exists","effect":"NoSchedule"}]` | Operator's tolerations | +| `operator.nodeSelector` | object | `{}` | Operator's node selector | +| `operator.extraEnv` | map[string]string | `{}` | Custom extra environment variables for the operator container | +| `operator.affinity` | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":1,"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/master","operator":"In","values":[""]}]}},{"weight":1,"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"In","values":[""]}]}}]}}` | Operator's afffinity configuration | +| `operator.nameOverride` | string | `` | Operator's resource name override | +| `operator.fullnameOverride` | string | `` | Operator's resource full name override | +| `operator.resourcePrefix` | string | `openshift.io` | Device plugin resource prefix | +| `operator.cniBinPath` | string | `/opt/cni/bin` | Path for CNI binary | +| `operator.clustertype` | string | `kubernetes` | Cluster environment type | +| `operator.metricsExporter.port` | string | `9110` | Port where the Network Metrics Exporter listen | +| `operator.metricsExporter.certificates.secretName` | string | `metrics-exporter-cert` | Secret name to serve metrics via TLS. The secret must have the same fields as `operator.admissionControllers.certificates.secretNames` | +| `operator.metricsExporter.prometheusOperator.enabled` | bool | false | Wheter the operator shoud configure Prometheus resources or not (e.g. `ServiceMonitors`). | +| `operator.metricsExporter.prometheusOperator.serviceAccount` | string | `prometheus-k8s` | The service account used by the Prometheus Operator. This is used to give Prometheus the permission to list resource in the SR-IOV operator namespace | +| `operator.metricsExporter.prometheusOperator.namespace` | string | `monitoring` | The namespace where the Prometheus Operator is installed. Setting this variable makes the operator deploy `monitoring.coreos.com` resources. | +| `operator.metricsExporter.prometheusOperator.deployRules` | bool | false | Whether the operator should deploy `PrometheusRules` to scrape namespace version of metrics. | + +#### Admission Controllers parameters + +The admission controllers can be enabled by switching on a single parameter `operator.admissionControllers.enabled`. By +default, the user needs to pre-create Kubernetes Secrets that match the names provided in +`operator.admissionControllers.certificates.secretNames`. The secrets should have 3 fields populated with the relevant +content: +* `ca.crt` (value needs to be base64 encoded twice) +* `tls.crt` +* `tls.key` + +Aside from the aforementioned mode, the chart supports 3 more modes for certificate consumption by the admission +controllers, which can be found in the table below. In a nutshell, the modes that are supported are: +* Consume pre-created Certificates managed by cert-manager +* Generate self signed Certificates managed by cert-manager +* Specify the content of the certificates as Helm values + +| Name | Type | Default | description | +| ---- | ---- | ------- | ----------- | +| `operator.admissionControllers.enabled` | bool | false | Flag that switches on the admission controllers | +| `operator.admissionControllers.certificates.secretNames.operator` | string | `operator-webhook-cert` | Secret that stores the certificate for the Operator's admission controller | +| `operator.admissionControllers.certificates.secretNames.injector` | string | `network-resources-injector-cert` | Secret that stores the certificate for the Network Resources Injector's admission controller | +| `operator.admissionControllers.certificates.certManager.enabled` | bool | false | Flag that switches on consumption of certificates managed by cert-manager | +| `operator.admissionControllers.certificates.certManager.generateSelfSigned` | bool | false | Flag that switches on generation of self signed certificates managed by cert-manager. The secrets in which the certificates are stored will have the names provided in `operator.admissionControllers.certificates.secretNames` | +| `operator.admissionControllers.certificates.custom.enabled` | bool | false | Flag that switches on consumption of user provided certificates that are part of `operator.admissionControllers.certificates.custom.operator` and `operator.admissionControllers.certificates.custom.injector` objects | +| `operator.admissionControllers.certificates.custom.operator.caCrt` | string | `` | The CA certificate to be used by the Operator's admission controller | +| `operator.admissionControllers.certificates.custom.operator.tlsCrt` | string | `` | The public part of the certificate to be used by the Operator's admission controller | +| `operator.admissionControllers.certificates.custom.operator.tlsKey` | string | `` | The private part of the certificate to be used by the Operator's admission controller | +| `operator.admissionControllers.certificates.custom.injector.caCrt` | string | `` | The CA certificate to be used by the Network Resources Injector's admission controller | +| `operator.admissionControllers.certificates.custom.injector.tlsCrt` | string | `` | The public part of the certificate to be used by the Network Resources Injector's admission controller | +| `operator.admissionControllers.certificates.custom.injector.tlsKey` | string | `` | The private part of the certificate to be used by the Network Resources Injector's admission controller | + +### SR-IOV Operator Configuration Parameters + +This section contains general parameters that apply to both the operator and daemon componets of SR-IOV Network Operator. + +| Name | Type | Default | description | +| ---- | ---- | ------- | ----------- | +| `sriovOperatorConfig.deploy` | bool | `false` | deploy SriovOperatorConfig custom resource | +| `sriovOperatorConfig.configDaemonNodeSelector` | map[string]string | `{}` | node selectors for sriov-network-config-daemon | +| `sriovOperatorConfig.logLevel` | int | `2` | log level for both operator and sriov-network-config-daemon | +| `sriovOperatorConfig.disableDrain` | bool | `false` | disable node draining when configuring SR-IOV, set to true in case of a single node cluster or any other justifiable reason | +| `sriovOperatorConfig.configurationMode` | string | `daemon` | sriov-network-config-daemon configuration mode. either `daemon` or `systemd` | +| `sriovOperatorConfig.disablePlugins` | list | `[]` | list of sriov-network-config-daemon plugins to disable (e.g., `["mellanox"]`) | +| `sriovOperatorConfig.featureGates` | map[string]bool | `{}` | feature gates to enable/disable | +| `sriovOperatorConfig.configDaemonEnvVars` | map[string]string | `{}` | custom environment variables for sriov-network-config-daemon | + +**Note** + +When `sriovOperatorConfig.configurationMode` is configured as `systemd`, configurations files and `systemd` service files are created on the node. +Upon chart deletion, those files are not cleaned up. For cases where this is not acceptable, users should rather configured the `daemon` mode. + +### Images parameters + +| Name | description | +| ---- | ----------- | +| `images.operator` | Operator controller image | +| `images.sriovConfigDaemon` | Daemon node agent image | +| `images.sriovCni` | SR-IOV CNI image | +| `images.ibSriovCni` | InfiniBand SR-IOV CNI image | +| `images.ovsCni` | OVS CNI image | +| `images.rdmaCni` | RDMA CNI image | +| `images.sriovDevicePlugin` | SR-IOV device plugin image | +| `images.resourcesInjector` | Resources Injector image | +| `images.webhook` | Operator Webhook image | +| `images.metricsExporter` | Network Metrics Exporter image | +| `images.metricsExporterKubeRbacProxy` | Kube RBAC Proxy image used for metrics exporter | + +### Extra objects parameters + +**Disclaimer**: + +Please note that any resources deployed using the `extraDeploy` in this Helm chart are the sole responsibility of the user. It is important to review and understand the implications of these deployed resources. The maintainers of this Helm chart take no responsibility for any issues or damages caused by the deployment or operation of these resources. + +| Name | description | +| ---- | ------------| +|`extraDeploy`| Array of extra objects to deploy with the release | diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_deviceattributes.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_deviceattributes.yaml new file mode 100644 index 00000000..c4ec7087 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_deviceattributes.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: deviceattributes.sriovnetwork.k8snetworkplumbingwg.io +spec: + group: sriovnetwork.k8snetworkplumbingwg.io + names: + kind: DeviceAttributes + listKind: DeviceAttributesList + plural: deviceattributes + singular: deviceattributes + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + DeviceAttributes defines a set of arbitrary attributes that can be applied + to devices selected by a SriovResourcePolicy. Policies reference + DeviceAttributes objects via label selectors. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: DeviceAttributesSpec holds the attributes to apply to devices. + properties: + attributes: + additionalProperties: + description: DeviceAttribute must have exactly one field set. + properties: + bool: + description: BoolValue is a true/false value. + type: boolean + int: + description: IntValue is a number. + format: int64 + type: integer + string: + description: StringValue is a string. Must not be longer than + 64 characters. + type: string + version: + description: |- + VersionValue is a semantic version according to semver.org spec 2.0.0. + Must not be longer than 64 characters. + type: string + type: object + description: |- + Attributes is a map of qualified attribute name to value. + Keys should be fully qualified (e.g. "k8s.cni.cncf.io/resourceName"). + type: object + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_sriovresourcepolicies.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_sriovresourcepolicies.yaml new file mode 100644 index 00000000..bd41a9d8 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.k8snetworkplumbingwg.io_sriovresourcepolicies.yaml @@ -0,0 +1,228 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovresourcepolicies.sriovnetwork.k8snetworkplumbingwg.io +spec: + group: sriovnetwork.k8snetworkplumbingwg.io + names: + kind: SriovResourcePolicy + listKind: SriovResourcePolicyList + plural: sriovresourcepolicies + singular: sriovresourcepolicy + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SriovResourcePolicy defines a policy for advertising SR-IOV devices as Kubernetes resources. + Devices matching the policy's resource filters are advertised in the ResourceSlice. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovResourcePolicySpec is the spec for a SriovResourcePolicy + properties: + configs: + items: + description: |- + Config pairs a device selection (ResourceFilters) with an optional set of + extra attributes to apply (DeviceAttributesSelector). Devices matching the + filters are advertised regardless of whether a DeviceAttributesSelector is set. + properties: + deviceAttributesSelector: + description: |- + DeviceAttributesSelector selects DeviceAttributes objects by label. + Attributes from all matching DeviceAttributes are merged and applied + to devices selected by ResourceFilters. Optional. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + resourceFilters: + items: + description: ResourceFilter is a filter for a resource + properties: + devices: + items: + type: string + type: array + drivers: + items: + type: string + type: array + pciAddresses: + items: + type: string + type: array + pfNames: + items: + type: string + type: array + pfPciAddresses: + items: + type: string + type: array + vendors: + items: + type: string + type: array + type: object + type: array + type: object + type: array + nodeSelector: + description: |- + A node selector represents the union of the results of one or more label queries + over a set of nodes; that is, it represents the OR of the selectors represented + by the node selector terms. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms + are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by node's + labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by node's + fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml new file mode 100644 index 00000000..63da95e4 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: ovsnetworks.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: OVSNetwork + listKind: OVSNetworkList + plural: ovsnetworks + singular: ovsnetwork + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: OVSNetwork is the Schema for the ovsnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: OVSNetworkSpec defines the desired state of OVSNetwork + properties: + bridge: + description: |- + name of the OVS bridge, if not set OVS will automatically select bridge + based on VF PCI address + type: string + capabilities: + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' + type: string + interfaceType: + description: The type of interface on ovs. + type: string + ipam: + description: IPAM configuration to be used for this network. + type: string + metaPlugins: + description: MetaPluginsConfig configuration to be used in order to + chain metaplugins + type: string + mtu: + description: Mtu for the OVS port + type: integer + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: OVS Network device plugin endpoint resource name + type: string + trunk: + description: Trunk configuration for the OVS port + items: + description: TrunkConfig contains configuration for bridge trunk + properties: + id: + maximum: 4095 + minimum: 0 + type: integer + maxID: + maximum: 4095 + minimum: 0 + type: integer + minID: + maximum: 4095 + minimum: 0 + type: integer + type: object + type: array + vlan: + description: Vlan to assign for the OVS port + maximum: 4095 + minimum: 0 + type: integer + required: + - resourceName + type: object + status: + description: OVSNetworkStatus defines the observed state of OVSNetwork + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml new file mode 100644 index 00000000..93e62764 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovibnetworks.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovIBNetwork + listKind: SriovIBNetworkList + plural: sriovibnetworks + singular: sriovibnetwork + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovIBNetwork is the Schema for the sriovibnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork + properties: + capabilities: + description: |- + Capabilities to be configured for this network. + Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}' + type: string + ipam: + description: IPAM configuration to be used for this network. + type: string + linkState: + description: VF link state (enable|disable|auto) + enum: + - auto + - enable + - disable + type: string + metaPlugins: + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: SRIOV Network device plugin endpoint resource name + type: string + required: + - resourceName + type: object + status: + description: SriovIBNetworkStatus defines the observed state of SriovIBNetwork + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml new file mode 100644 index 00000000..66b32f00 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -0,0 +1,265 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovnetworknodepolicies.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkNodePolicy + listKind: SriovNetworkNodePolicyList + plural: sriovnetworknodepolicies + singular: sriovnetworknodepolicy + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkNodePolicy is the Schema for the sriovnetworknodepolicies + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkNodePolicySpec defines the desired state of SriovNetworkNodePolicy + properties: + bridge: + description: |- + contains bridge configuration for matching PFs, + valid only for eSwitchMode==switchdev + properties: + groupingPolicy: + description: contains OVS bridges grouping configuration + enum: + - perPF + - perNIC + - all + type: string + ovs: + description: contains configuration for the OVS bridge, + properties: + bridge: + description: contains bridge level settings + properties: + datapathType: + description: configure datapath_type field in the Bridge + table in OVSDB + type: string + externalIDs: + additionalProperties: + type: string + description: IDs to inject to external_ids field in the + Bridge table in OVSDB + type: object + failMode: + description: configure fail_mode field in the Bridge table + in OVSDB (optional). 'secure' or 'standalone'. + enum: + - secure + - standalone + type: string + otherConfig: + additionalProperties: + type: string + description: additional options to inject to other_config + field in the bridge table in OVSDB + type: object + type: object + uplink: + description: contains settings for uplink (PF) + properties: + interface: + description: contains settings for PF interface in the + OVS bridge + properties: + externalIDs: + additionalProperties: + type: string + description: external_ids field in the Interface table + in OVSDB + type: object + mtuRequest: + description: mtu_request field in the Interface table + in OVSDB + type: integer + options: + additionalProperties: + type: string + description: options field in the Interface table + in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: other_config field in the Interface table + in OVSDB + type: object + type: + description: type field in the Interface table in + OVSDB + type: string + type: object + type: object + type: object + type: object + deviceType: + default: netdevice + description: The driver type for configured VFs. Allowed value "netdevice", + "vfio-pci". Defaults to netdevice. + enum: + - netdevice + - vfio-pci + type: string + devlinkParams: + description: contains devlink params for NIC devices + properties: + params: + items: + description: DevlinkParam defines the parameter for devlink + configuration + properties: + applyOn: + description: 'Device to apply devlink parameter: PF (default)|VF|SF' + enum: + - PF + - pf + - VF + - vf + - SF + - sf + type: string + cmode: + description: 'cmode option: runtime (default) | driverinit + | permanent (runtime is only supported now)' + enum: + - runtime + - driverinit + - permanent + type: string + name: + description: Param name + type: string + value: + description: Param value + type: string + type: object + type: array + type: object + eSwitchMode: + description: NIC Device Mode. Allowed value "legacy","switchdev". + enum: + - legacy + - switchdev + type: string + excludeTopology: + description: Exclude device's NUMA node when advertising this resource + by SRIOV network device plugin. Default to false. + type: boolean + externallyManaged: + description: don't create the virtual function only allocated them + to the device plugin. Defaults to false. + type: boolean + isRdma: + description: RDMA mode. Defaults to false. + type: boolean + linkType: + description: NIC Link Type. Allowed value "eth", "ETH", "ib", and + "IB". + enum: + - eth + - ETH + - ib + - IB + type: string + mtu: + description: MTU of VF + minimum: 1 + type: integer + needVhostNet: + description: mount vhost-net device. Defaults to false. + type: boolean + nicSelector: + description: NicSelector selects the NICs to be configured + properties: + deviceID: + description: The device hex code of SR-IoV device. Allowed value + "0d58", "1572", "158b", "1013", "1015", "1017", "101b". + type: string + netFilter: + description: |- + Infrastructure Networking selection filter. + Allowed values: + - "openstack/NetworkID:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + - "aws/NetworkID:xxxxxxxx" + type: string + pfNames: + description: Name of SR-IoV PF. + items: + type: string + type: array + rootDevices: + description: PCI address of SR-IoV PF. + items: + type: string + type: array + vendor: + description: The vendor hex code of SR-IoV device. Allowed value + "8086", "15b3". + type: string + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes to be configured + type: object + numVfs: + description: Number of VFs for each PF + minimum: 0 + type: integer + priority: + description: Priority of the policy, higher priority policies can + override lower ones. + maximum: 99 + minimum: 0 + type: integer + resourceName: + description: SRIOV Network device plugin endpoint resource name + type: string + vdpaType: + description: VDPA device type. Allowed value "virtio", "vhost" + enum: + - virtio + - vhost + type: string + required: + - nicSelector + - nodeSelector + - numVfs + - resourceName + type: object + status: + description: SriovNetworkNodePolicyStatus defines the observed state of + SriovNetworkNodePolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml new file mode 100644 index 00000000..d7fe8142 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -0,0 +1,529 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovnetworknodestates.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkNodeState + listKind: SriovNetworkNodeStateList + plural: sriovnetworknodestates + singular: sriovnetworknodestate + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.syncStatus + name: Sync Status + type: string + - jsonPath: .metadata.annotations.sriovnetwork\.openshift\.io/desired-state + name: Desired Sync State + type: string + - jsonPath: .metadata.annotations.sriovnetwork\.openshift\.io/current-state + name: Current Sync State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkNodeState is the Schema for the sriovnetworknodestates + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkNodeStateSpec defines the desired state of SriovNetworkNodeState + properties: + bridges: + description: Bridges contains list of bridges + properties: + groupingPolicy: + description: contains OVS bridges grouping configuration + enum: + - perPF + - perNIC + - all + type: string + ovs: + items: + description: OVSConfigExt contains configuration for the concrete + OVS bridge + properties: + bridge: + description: bridge-level configuration for the bridge + properties: + datapathType: + description: configure datapath_type field in the Bridge + table in OVSDB + type: string + externalIDs: + additionalProperties: + type: string + description: IDs to inject to external_ids field in + the Bridge table in OVSDB + type: object + failMode: + description: configure fail_mode field in the Bridge + table in OVSDB (optional). 'secure' or 'standalone'. + enum: + - secure + - standalone + type: string + otherConfig: + additionalProperties: + type: string + description: additional options to inject to other_config + field in the bridge table in OVSDB + type: object + type: object + name: + description: name of the bridge + type: string + uplinks: + description: |- + uplink-level bridge configuration for each uplink(PF). + currently must contain only one element + items: + description: OVSUplinkConfigExt contains configuration + for the concrete OVS uplink(PF) + properties: + interface: + description: configuration from the Interface OVS + table for the PF + properties: + externalIDs: + additionalProperties: + type: string + description: external_ids field in the Interface + table in OVSDB + type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer + options: + additionalProperties: + type: string + description: options field in the Interface table + in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: other_config field in the Interface + table in OVSDB + type: object + type: + description: type field in the Interface table + in OVSDB + type: string + type: object + name: + description: name of the PF interface + type: string + pciAddress: + description: pci address of the PF + type: string + required: + - pciAddress + type: object + type: array + required: + - name + type: object + type: array + type: object + interfaces: + items: + properties: + devlinkParams: + description: DevlinkParams defines the parameters for devlink + configuration + properties: + params: + items: + description: DevlinkParam defines the parameter for devlink + configuration + properties: + applyOn: + description: 'Device to apply devlink parameter: PF + (default)|VF|SF' + enum: + - PF + - pf + - VF + - vf + - SF + - sf + type: string + cmode: + description: 'cmode option: runtime (default) | driverinit + | permanent (runtime is only supported now)' + enum: + - runtime + - driverinit + - permanent + type: string + name: + description: Param name + type: string + value: + description: Param value + type: string + type: object + type: array + type: object + eSwitchMode: + type: string + externallyManaged: + type: boolean + linkType: + type: string + mtu: + type: integer + name: + type: string + numVfs: + type: integer + pciAddress: + type: string + vfGroups: + items: + properties: + deviceType: + type: string + isRdma: + type: boolean + mtu: + type: integer + policyName: + type: string + resourceName: + type: string + vdpaType: + type: string + vfRange: + type: string + type: object + type: array + required: + - pciAddress + type: object + type: array + system: + properties: + ovsConfig: + additionalProperties: + type: string + default: + hw-offload: "true" + description: OVS config. It will be provided for ovs-vswitchd + service as other_config option + type: object + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object + type: object + status: + description: SriovNetworkNodeStateStatus defines the observed state of + SriovNetworkNodeState + properties: + bridges: + description: Bridges contains list of bridges + properties: + groupingPolicy: + description: contains OVS bridges grouping configuration + enum: + - perPF + - perNIC + - all + type: string + ovs: + items: + description: OVSConfigExt contains configuration for the concrete + OVS bridge + properties: + bridge: + description: bridge-level configuration for the bridge + properties: + datapathType: + description: configure datapath_type field in the Bridge + table in OVSDB + type: string + externalIDs: + additionalProperties: + type: string + description: IDs to inject to external_ids field in + the Bridge table in OVSDB + type: object + failMode: + description: configure fail_mode field in the Bridge + table in OVSDB (optional). 'secure' or 'standalone'. + enum: + - secure + - standalone + type: string + otherConfig: + additionalProperties: + type: string + description: additional options to inject to other_config + field in the bridge table in OVSDB + type: object + type: object + name: + description: name of the bridge + type: string + uplinks: + description: |- + uplink-level bridge configuration for each uplink(PF). + currently must contain only one element + items: + description: OVSUplinkConfigExt contains configuration + for the concrete OVS uplink(PF) + properties: + interface: + description: configuration from the Interface OVS + table for the PF + properties: + externalIDs: + additionalProperties: + type: string + description: external_ids field in the Interface + table in OVSDB + type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer + options: + additionalProperties: + type: string + description: options field in the Interface table + in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: other_config field in the Interface + table in OVSDB + type: object + type: + description: type field in the Interface table + in OVSDB + type: string + type: object + name: + description: name of the PF interface + type: string + pciAddress: + description: pci address of the PF + type: string + required: + - pciAddress + type: object + type: array + required: + - name + type: object + type: array + type: object + interfaces: + items: + properties: + Vfs: + items: + properties: + Vlan: + type: integer + assigned: + type: string + deviceID: + type: string + devlinkParams: + description: DevlinkParams defines the parameters for + devlink configuration + properties: + params: + items: + description: DevlinkParam defines the parameter + for devlink configuration + properties: + applyOn: + description: 'Device to apply devlink parameter: + PF (default)|VF|SF' + enum: + - PF + - pf + - VF + - vf + - SF + - sf + type: string + cmode: + description: 'cmode option: runtime (default) + | driverinit | permanent (runtime is only + supported now)' + enum: + - runtime + - driverinit + - permanent + type: string + name: + description: Param name + type: string + value: + description: Param value + type: string + type: object + type: array + type: object + driver: + type: string + guid: + type: string + mac: + type: string + mtu: + type: integer + name: + type: string + pciAddress: + type: string + representorName: + type: string + vdpaType: + type: string + vendor: + type: string + vfID: + type: integer + required: + - pciAddress + - vfID + type: object + type: array + altNames: + items: + type: string + type: array + deviceID: + type: string + devlinkParams: + description: DevlinkParams defines the parameters for devlink + configuration + properties: + params: + items: + description: DevlinkParam defines the parameter for devlink + configuration + properties: + applyOn: + description: 'Device to apply devlink parameter: PF + (default)|VF|SF' + enum: + - PF + - pf + - VF + - vf + - SF + - sf + type: string + cmode: + description: 'cmode option: runtime (default) | driverinit + | permanent (runtime is only supported now)' + enum: + - runtime + - driverinit + - permanent + type: string + name: + description: Param name + type: string + value: + description: Param value + type: string + type: object + type: array + type: object + driver: + type: string + eSwitchMode: + type: string + externallyManaged: + type: boolean + linkAdminState: + type: string + linkSpeed: + type: string + linkType: + type: string + mac: + type: string + mtu: + type: integer + name: + type: string + netFilter: + type: string + numVfs: + type: integer + pciAddress: + type: string + totalvfs: + type: integer + vendor: + type: string + required: + - pciAddress + type: object + type: array + lastSyncError: + type: string + syncStatus: + type: string + system: + properties: + ovsConfig: + additionalProperties: + type: string + default: + hw-offload: "true" + description: OVS config. It will be provided for ovs-vswitchd + service as other_config option + type: object + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml new file mode 100644 index 00000000..bce2dec7 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -0,0 +1,138 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkPoolConfig + listKind: SriovNetworkPoolConfigList + plural: sriovnetworkpoolconfigs + singular: sriovnetworkpoolconfig + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkPoolConfig is the Schema for the sriovnetworkpoolconfigs + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkPoolConfigSpec defines the desired state of SriovNetworkPoolConfig + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + maxUnavailable defines either an integer number or percentage + of nodes in the pool that can go Unavailable during an update. + + A value larger than 1 will mean multiple nodes going unavailable during + the update, which may affect your workload stress on the remaining nodes. + Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, + even if maxUnavailable is greater than one. + x-kubernetes-int-or-string: true + nodeSelector: + description: nodeSelector specifies a label selector for Nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + ovsHardwareOffloadConfig: + description: OvsHardwareOffloadConfig describes the OVS HWOL configuration + for selected Nodes + properties: + name: + description: |- + Name is mandatory and must be unique. + On Kubernetes: + Name is the name of OvsHardwareOffloadConfig + On OpenShift: + Name is the name of MachineConfigPool to be enabled with OVS hardware offload + type: string + otherConfig: + additionalProperties: + type: string + default: + hw-offload: "true" + description: OVS config. It will be provided for ovs-vswitchd + service as other_config option + type: object + type: object + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object + status: + description: SriovNetworkPoolConfigStatus defines the observed state of + SriovNetworkPoolConfig + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml new file mode 100644 index 00000000..29e686b3 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovnetworks.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetwork + listKind: SriovNetworkList + plural: sriovnetworks + singular: sriovnetwork + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetwork is the Schema for the sriovnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkSpec defines the desired state of SriovNetwork + properties: + capabilities: + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' + type: string + ipam: + description: IPAM configuration to be used for this network. + type: string + linkState: + description: VF link state (enable|disable|auto) + enum: + - auto + - enable + - disable + type: string + logFile: + description: |- + LogFile sets the log file of the SRIOV CNI plugin logs. If unset (default), this will log to stderr and thus + to multus and container runtime logs. + type: string + logLevel: + default: info + description: |- + LogLevel sets the log level of the SRIOV CNI plugin - either of panic, error, warning, info, debug. Defaults + to info if left blank. + enum: + - panic + - error + - warning + - info + - debug + - "" + type: string + maxTxRate: + description: Maximum tx rate, in Mbps, for the VF. Defaults to 0 (no + rate limiting) + minimum: 0 + type: integer + metaPlugins: + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. + type: string + minTxRate: + description: Minimum tx rate, in Mbps, for the VF. Defaults to 0 (no + rate limiting). min_tx_rate should be <= max_tx_rate. + minimum: 0 + type: integer + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: SRIOV Network device plugin endpoint resource name + type: string + spoofChk: + description: VF spoof check, (on|off) + enum: + - "on" + - "off" + type: string + trust: + description: VF trust mode (on|off) + enum: + - "on" + - "off" + type: string + vlan: + description: VLAN ID to assign for the VF. Defaults to 0. + maximum: 4094 + minimum: 0 + type: integer + vlanProto: + description: VLAN proto to assign for the VF. Defaults to 802.1q. + enum: + - 802.1q + - 802.1Q + - 802.1ad + - 802.1AD + type: string + vlanQoS: + description: VLAN QoS ID to assign for the VF. Defaults to 0. + maximum: 7 + minimum: 0 + type: integer + required: + - resourceName + type: object + status: + description: SriovNetworkStatus defines the observed state of SriovNetwork + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml new file mode 100644 index 00000000..29575139 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -0,0 +1,121 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.0 + name: sriovoperatorconfigs.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovOperatorConfig + listKind: SriovOperatorConfigList + plural: sriovoperatorconfigs + singular: sriovoperatorconfig + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovOperatorConfig is the Schema for the sriovoperatorconfigs + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovOperatorConfigSpec defines the desired state of SriovOperatorConfig + properties: + configDaemonEnvVars: + additionalProperties: + type: string + description: |- + ConfigDaemonEnvVars allows to specify custom environment variables + for the sriov-network-config-daemon + type: object + configDaemonNodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes to be configured + type: object + configurationMode: + description: |- + Flag to enable the sriov-network-config-daemon to use a systemd service to configure SR-IOV devices on boot + Default mode: daemon + enum: + - daemon + - systemd + type: string + disableDrain: + description: Flag to disable nodes drain during debugging + type: boolean + disablePlugins: + description: DisablePlugins is a list of sriov-network-config-daemon + plugins to disable + items: + description: PluginNameValue defines the plugin name + enum: + - mellanox + type: string + type: array + enableInjector: + description: Flag to control whether the network resource injector + webhook shall be deployed + type: boolean + enableOperatorWebhook: + description: Flag to control whether the operator admission controller + webhook shall be deployed + type: boolean + enableOvsOffload: + description: Flag to enable OVS hardware offload. Set to 'true' to + provision switchdev-configuration.service and enable OpenvSwitch + hw-offload on nodes. + type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates to enable experimental features + type: object + logLevel: + description: Flag to control the log verbose level of the operator. + Set to '0' to show only the basic logs. And set to '2' to show all + the available logs. + maximum: 2 + minimum: 0 + type: integer + useCDI: + description: Flag to enable Container Device Interface mode for SR-IOV + Network Device Plugin + type: boolean + type: object + status: + description: SriovOperatorConfigStatus defines the observed state of SriovOperatorConfig + properties: + injector: + description: Show the runtime status of the network resource injector + webhook + type: string + operatorWebhook: + description: Show the runtime status of the operator admission controller + webhook + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/NOTES.txt b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/NOTES.txt new file mode 100644 index 00000000..94dcf311 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/NOTES.txt @@ -0,0 +1,6 @@ +Get Network Operator deployed resources by running the following commands: + +$ kubectl -n {{ .Release.Namespace }} get pods + +For additional instructions on how to use SR-IOV network operator, +refer to: https://github.com/k8snetworkplumbingwg/sriov-network-operator \ No newline at end of file diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/_helpers.tpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/_helpers.tpl new file mode 100644 index 00000000..2cde6817 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "sriov-network-operator.name" -}} +{{- default "sriov-network-operator" .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sriov-network-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default "sriov-network-operator" .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "sriov-network-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "sriov-network-operator.labels" -}} +helm.sh/chart: {{ include "sriov-network-operator.chart" . }} +{{ include "sriov-network-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "sriov-network-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sriov-network-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "sriov-network-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "sriov-network-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/certificate.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/certificate.yaml new file mode 100644 index 00000000..add29a9b --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/certificate.yaml @@ -0,0 +1,71 @@ +{{- if .Values.operator.admissionControllers.enabled }} +{{- if and (.Values.operator.admissionControllers.certificates.certManager.enabled) (.Values.operator.admissionControllers.certificates.certManager.generateSelfSigned) }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - operator-webhook-service.{{ .Release.Namespace }}.svc + - operator-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: operator-webhook-selfsigned-issuer + secretName: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: operator-webhook-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - network-resources-injector-service.{{ .Release.Namespace }}.svc + - network-resources-injector-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: network-resources-injector-selfsigned-issuer + secretName: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: network-resources-injector-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- else if and (not .Values.operator.admissionControllers.certificates.certManager.enabled) (.Values.operator.admissionControllers.certificates.custom.enabled) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.operator.admissionControllers.certificates.custom.operator.caCrt | b64enc | b64enc | quote }} + tls.crt: {{ .Values.operator.admissionControllers.certificates.custom.operator.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionControllers.certificates.custom.operator.tlsKey | b64enc | quote }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.operator.admissionControllers.certificates.custom.injector.caCrt | b64enc | b64enc | quote }} + tls.crt: {{ .Values.operator.admissionControllers.certificates.custom.injector.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionControllers.certificates.custom.injector.tlsKey | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml new file mode 100644 index 00000000..13d5bff9 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml @@ -0,0 +1,60 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["*"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["namespaces", "serviceaccounts"] + verbs: ["*"] + - apiGroups: ["k8s.cni.cncf.io"] + resources: ["network-attachment-definitions"] + verbs: ["*"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: [clusterroles, clusterrolebindings] + verbs: ["*"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["*"] + - apiGroups: ["sriovnetwork.openshift.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["machineconfiguration.openshift.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["config.openshift.io"] + resources: ["infrastructures", "apiservers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["resource.k8s.io"] + resources: ["deviceclasses"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: ["sriovnetwork.k8snetworkplumbingwg.io"] + resources: ["sriovresourcepolicies", "deviceattributes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sriov-network-config-daemon + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: [ "config.openshift.io" ] + resources: [ "infrastructures" ] + verbs: [ "get", "list", "watch" ] diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..c10aa9be --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml @@ -0,0 +1,29 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +roleRef: + kind: ClusterRole + name: {{ include "sriov-network-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: {{ include "sriov-network-operator.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sriov-network-config-daemon + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +roleRef: + kind: ClusterRole + name: sriov-network-config-daemon + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: sriov-network-config-daemon diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/configmap.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/configmap.yaml new file mode 100644 index 00000000..8f459243 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/configmap.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supported-nic-ids +data: + Intel_i40e_XXV710: "8086 158a 154c" + Intel_i40e_25G_SFP28: "8086 158b 154c" + Intel_i40e_10G_X710_SFP: "8086 1572 154c" + Intel_ixgbe_10G_X550: "8086 1563 1565" + Intel_ixgbe_82576: "8086 10c9 10ca" + Intel_i40e_X710_X557_AT_10G: "8086 1589 154c" + Intel_i40e_10G_X710_BACKPLANE: "8086 1581 154c" + Intel_i40e_10G_X710_BASE_T: "8086 15ff 154c" + Intel_i40e_XXV710_N3000: "8086 0d58 154c" + Intel_i40e_40G_XL710_QSFP: "8086 1583 154c" + Intel_ice_Columbiaville_E810-CQDA2_2CQDA2: "8086 1592 1889" + Intel_ice_Columbiaville_E810-XXVDA4: "8086 1593 1889" + Intel_ice_Columbiaville_E810-XXVDA2: "8086 159b 1889" + Intel_ice_Columbiaville_E810-XXV_BACKPLANE: "8086 1599 1889" + Intel_ice_Columbiaville_E810: "8086 1591 1889" + Intel_ice_Columbiapark_E823C: "8086 188a 1889" + Intel_ice_Columbiapark_E823L_SFP: "8086 124d 1889" + Intel_ice_Columbiapark_E823L_BACKPLANE: "8086 124c 1889" + Intel_ice_Columbiapark_E825C_BACKPLANE: "8086 579c 1889" + Intel_ice_Columbiapark_E825C_QSFP: "8086 579d 1889" + Intel_ice_Columbiapark_E825C_SFP: "8086 579e 1889" + Intel_ice_Connorsville_E830_QSFP: "8086 12d2 1889" + Intel_ice_Connorsville_E830_SFP: "8086 12d3 1889" + Intel_ice_Connorsville_E835CC_QSFP: "8086 1249 1889" + Intel_ice_Connorsville_E835CC_SFP: "8086 124a 1889" + Nvidia_mlx5_ConnectX-4: "15b3 1013 1014" + Nvidia_mlx5_ConnectX-4LX: "15b3 1015 1016" + Nvidia_mlx5_ConnectX-5: "15b3 1017 1018" + Nvidia_mlx5_ConnectX-5_Ex: "15b3 1019 101a" + Nvidia_mlx5_ConnectX-6: "15b3 101b 101c" + Nvidia_mlx5_ConnectX-6_Dx: "15b3 101d 101e" + Nvidia_mlx5_ConnectX-6_Lx: "15b3 101f 101e" + Nvidia_mlx5_ConnectX-7: "15b3 1021 101e" + Nvidia_mlx5_ConnectX-8: "15b3 1023 101e" + Nvidia_mlx5_ConnectX-9: "15b3 1025 101e" + Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx: "15b3 a2d6 101e" + Nvidia_mlx5_MT43244_BlueField-3_integrated_ConnectX-7_Dx: "15b3 a2dc 101e" + Broadcom_bnxt_BCM57414_2x25G: "14e4 16d7 16dc" + Broadcom_bnxt_BCM75508_2x100G: "14e4 1750 1806" + Qlogic_qede_QL45000_50G: "1077 1654 1664" + Red_Hat_Virtio_network_device: "1af4 1000 1000" + Red_Hat_Virtio_1_0_network_device: "1af4 1041 1041" + Marvell_OCTEON_TX2_CN96XX: "177d b200 b203" + Marvell_OCTEON_TX2_CN98XX: "177d b100 b103" + Marvell_OCTEON_Fusion_CNF95XX: "177d b600 b603" + Marvell_OCTEON10_CN10XXX: "177d b900 b903" + Marvell_OCTEON_Fusion_CNF105XX: "177d ba00 ba03" + Amazon_Elastic_Network_Adapter: "1d0f ec20 ec20" + {{- range .Values.supportedExtraNICs }} + {{ . }} + {{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/extra-list.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/extra-list.yaml new file mode 100644 index 00000000..6968e7db --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/extra-list.yaml @@ -0,0 +1,8 @@ +{{- range .Values.extraDeploy }} +--- +{{- if typeIs "string" . }} + {{- tpl . $ }} +{{- else }} + {{- tpl (. | toYaml) $ }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/operator.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/operator.yaml new file mode 100644 index 00000000..c0f34fb8 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/operator.yaml @@ -0,0 +1,168 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + name: sriov-network-operator + strategy: + type: Recreate + template: + metadata: + annotations: + openshift.io/required-scc: restricted-v2 + labels: + name: sriov-network-operator + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity }} + affinity: + {{- toYaml . | nindent 8}} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "sriov-network-operator.fullname" . }} + priorityClassName: "system-node-critical" + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: {{ include "sriov-network-operator.fullname" . }} + image: {{ .Values.images.operator }} + command: + - sriov-network-operator + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SRIOV_CNI_IMAGE + value: {{ .Values.images.sriovCni }} + - name: SRIOV_INFINIBAND_CNI_IMAGE + value: {{ .Values.images.ibSriovCni }} + - name: OVS_CNI_IMAGE + value: {{ .Values.images.ovsCni }} + - name: RDMA_CNI_IMAGE + value: {{ .Values.images.rdmaCni }} + - name: SRIOV_DEVICE_PLUGIN_IMAGE + value: {{ .Values.images.sriovDevicePlugin }} + - name: SRIOV_DRA_DRIVER_IMAGE + value: {{ .Values.images.sriovDraDriver | default "ghcr.io/k8snetworkplumbingwg/dra-driver-sriov:latest" }} + - name: DRA_DRIVER_CDI_ROOT + value: {{ .Values.draDriver.cdiRoot | default "/var/run/cdi" }} + - name: DRA_DRIVER_DEFAULT_INTERFACE_PREFIX + value: {{ .Values.draDriver.defaultInterfacePrefix | default "net" }} + - name: NETWORK_RESOURCES_INJECTOR_IMAGE + value: {{ .Values.images.resourcesInjector }} + - name: OPERATOR_NAME + value: sriov-network-operator + - name: SRIOV_NETWORK_CONFIG_DAEMON_IMAGE + value: {{ .Values.images.sriovConfigDaemon }} + - name: SRIOV_NETWORK_WEBHOOK_IMAGE + value: {{ .Values.images.webhook }} + - name: METRICS_EXPORTER_IMAGE + value: {{ .Values.images.metricsExporter }} + - name: METRICS_EXPORTER_PORT + value: "{{ .Values.operator.metricsExporter.port }}" + - name: METRICS_EXPORTER_SECRET_NAME + value: {{ .Values.operator.metricsExporter.certificates.secretName }} + - name: METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE + value: {{ .Values.images.metricsExporterKubeRbacProxy }} + {{- if .Values.operator.externalDrainer.enabled }} + - name: USE_EXTERNAL_DRAINER + value: {{ .Values.operator.externalDrainer.enabled | quote }} + {{- end }} + {{- if .Values.operator.metricsExporter.prometheusOperator.enabled }} + - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED + value: {{ .Values.operator.metricsExporter.prometheusOperator.enabled | quote}} + - name: METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES + value: {{ .Values.operator.metricsExporter.prometheusOperator.deployRules | quote}} + - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT + value: {{ .Values.operator.metricsExporter.prometheusOperator.serviceAccount }} + - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE + value: {{ .Values.operator.metricsExporter.prometheusOperator.namespace }} + {{- end }} + - name: RESOURCE_PREFIX + value: {{ .Values.operator.resourcePrefix }} + - name: IMAGE_PULL_SECRETS + value: {{ join "," .Values.imagePullSecrets }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: RELEASE_VERSION + value: {{ .Release.AppVersion }} + - name: SRIOV_CNI_BIN_PATH + value: {{ .Values.operator.cniBinPath }} + - name: CLUSTER_TYPE + value: {{ .Values.operator.clusterType }} + - name: STALE_NODE_STATE_CLEANUP_DELAY_MINUTES + value: "{{ .Values.operator.staleNodeStateCleanupDelayMinutes }}" + {{- range $k, $v := .Values.operator.extraEnv }} + - name: {{ $k }} + value: {{ $v | toString | quote }} + {{- end }} + - name: TLS_CIPHER_SUITES + value: {{ .Values.operator.tls.cipherSuites | quote }} + - name: TLS_MIN_VERSION + value: {{ .Values.operator.tls.minVersion | quote }} + - name: TLS_CURVE_PREFERENCES + value: {{ .Values.operator.tls.curvePreferences | quote }} + {{- if .Values.operator.admissionControllers.enabled }} + - name: OPERATOR_WEBHOOK_NETWORK_POLICY_PORT + value: "{{ .Values.operator.admissionControllers.networkPolicy.operator.port }}" + - name: INJECTOR_WEBHOOK_NETWORK_POLICY_PORT + value: "{{ .Values.operator.admissionControllers.networkPolicy.injector.port }}" + - name: ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_SECRET_NAME + value: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_SECRET_NAME + value: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + {{- if .Values.operator.admissionControllers.certificates.certManager.enabled }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED + value: {{ .Values.operator.admissionControllers.certificates.certManager.enabled | quote }} + {{- else }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT + valueFrom: + secretKeyRef: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + key: ca.crt + - name: ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT + valueFrom: + secretKeyRef: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + key: ca.crt + {{- end }} + {{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml new file mode 100644 index 00000000..5e2a47e6 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml @@ -0,0 +1,65 @@ +# The following job will be used as Helm pre-delete hook. It executes a small go-client binary +# which intent to delete 'default' SriovOperatorConfig, that triggers operator removal of generated cluster objects +# e.g. mutating/validating webhooks, within operator's recoinciling loop and +# preventing operator cluster object remainings while using helm uninstall +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "sriov-network-operator.fullname" . }}-pre-delete-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + spec: + serviceAccountName: {{ include "sriov-network-operator.fullname" . }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + containers: + - name: cleanup + image: {{ .Values.images.operator }} + securityContext: + capabilities: + drop: + - ALL + runAsNonRoot: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + command: + - sriov-network-operator-config-cleanup + args: + - --namespace + - {{ .Release.Namespace }} + restartPolicy: Never + backoffLimit: 2 + diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/role.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/role.yaml new file mode 100644 index 00000000..4d4b9d29 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/role.yaml @@ -0,0 +1,148 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - get + - create + - update + - delete + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + - prometheusrules + verbs: + - get + - create + - update + - delete + - apiGroups: + - apps + resourceNames: + - sriov-network-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - serviceaccounts + - roles + - rolebindings + verbs: + - '*' + - apiGroups: + - config.openshift.io + resources: + - infrastructures + verbs: + - get + - list + - watch + - apiGroups: + - 'coordination.k8s.io' + resources: + - 'leases' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - "get" + - "list" + - "watch" + - "delete" + - "patch" + - apiGroups: + - sriovnetwork.openshift.io + resources: + - '*' + - sriovnetworknodestates + verbs: + - '*' + - apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - update + - apiGroups: + - 'coordination.k8s.io' + resources: + - 'leases' + verbs: + - '*' + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator-webhook-sa + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml new file mode 100644 index 00000000..2450cc06 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml @@ -0,0 +1,31 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "sriov-network-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: sriov-network-config-daemon + apiGroup: rbac.authorization.k8s.io diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml new file mode 100644 index 00000000..fc0bb570 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml new file mode 100644 index 00000000..9c6731ce --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml @@ -0,0 +1,31 @@ +{{ if .Values.sriovOperatorConfig.deploy }} +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovOperatorConfig +metadata: + name: default + namespace: {{ .Release.Namespace }} +spec: + enableInjector: {{ .Values.operator.admissionControllers.enabled }} + enableOperatorWebhook: {{ .Values.operator.admissionControllers.enabled }} + {{- with .Values.sriovOperatorConfig.configDaemonNodeSelector }} + configDaemonNodeSelector: + {{- range $k, $v := .}}{{printf "%s: \"%s\"" $k $v | nindent 4 }}{{ end }} + {{- end }} + logLevel: {{ .Values.sriovOperatorConfig.logLevel }} + disableDrain: {{ .Values.sriovOperatorConfig.disableDrain }} + configurationMode: {{ .Values.sriovOperatorConfig.configurationMode }} + {{- with .Values.sriovOperatorConfig.disablePlugins }} + disablePlugins: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- with .Values.sriovOperatorConfig.featureGates }} + featureGates: + {{- range $k, $v := .}}{{printf "%s: %t" $k $v | nindent 4 }}{{ end }} + {{- end }} + {{- with .Values.sriovOperatorConfig.configDaemonEnvVars }} + configDaemonEnvVars: + {{- range $k, $v := .}}{{ printf "%s: %s" $k ($v | toString | quote) | nindent 4 }}{{ end }} + {{- end }} +{{ end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/values.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/values.yaml new file mode 100644 index 00000000..1ba72933 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/values.yaml @@ -0,0 +1,160 @@ +operator: + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: {} + # custom extra environment variables for the operator container + extraEnv: {} + # TLS configuration for admission webhooks and metrics proxy (Kubernetes only). + # On OpenShift, TLS settings are read from the config.openshift.io/v1 APIServer TLS profile. + # Leave empty to use Go defaults. + tls: + # Comma-separated list of TLS cipher suites (IANA names). + # Example: "TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" + cipherSuites: "" + # Minimum TLS version. + # Example: "VersionTLS12" or "VersionTLS13" + minVersion: "" + # Comma-separated list of TLS curve/group names for key exchange. + # Example: "X25519,secp256r1,secp384r1" + curvePreferences: "" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [""] + nameOverride: "" + fullnameOverride: "" + resourcePrefix: "openshift.io" + cniBinPath: "/opt/cni/bin" + clusterType: "kubernetes" + # minimal amount of time (in minutes) the operator will wait before removing + # stale SriovNetworkNodeState objects (objects that doesn't match node with the daemon) + # "0" means no extra delay, in this case the CR will be removed by the next reconcilation cycle (may take up to 5 minutes) + staleNodeStateCleanupDelayMinutes: "30" + metricsExporter: + port: "9110" + certificates: + secretName: "metrics-exporter-cert" + prometheusOperator: + enabled: false + serviceAccount: "prometheus-k8s" + namespace: "monitoring" + deployRules: false + # use external drain controller, for example utilizing NVIDIA maintenance operator + externalDrainer: + enabled: false + admissionControllers: + enabled: false + networkPolicy: + operator: + port: "6443" + injector: + port: "6443" + certificates: + secretNames: + operator: "operator-webhook-cert" + injector: "network-resources-injector-cert" + certManager: + # When enabled, makes use of certificates managed by cert-manager. + enabled: false + # When enabled, certificates are generated via cert-manager and then name will match the name of the secrets + # defined above + generateSelfSigned: false + # If not specified, no secret is created and secrets with the names defined above are expected to exist in the + # cluster. In that case, the ca.crt must be base64 encoded twice since it ends up being an env variable. + custom: + enabled: false + # operator: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + # injector: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + +sriovOperatorConfig: + # deploy sriovOperatorConfig CR with the below values + deploy: false + # node selectors for sriov-network-config-daemon + configDaemonNodeSelector: {} + # log level for both operator and sriov-network-config-daemon + logLevel: 2 + # disable node draining when configuring SR-IOV, set to true in case of a single node + # cluster or any other justifiable reason + disableDrain: false + # sriov-network-config-daemon configuration mode. either "daemon" or "systemd" + configurationMode: daemon + # list of sriov-network-config-daemon plugins to disable (e.g., ["mellanox"]) + disablePlugins: [] + # feature gates to enable/disable + featureGates: {} + # custom environment variables for sriov-network-config-daemon + configDaemonEnvVars: {} + +# Example for supportedExtraNICs values ['MyNIC: "8086 1521 1520"'] +supportedExtraNICs: [] + +# Image URIs for sriov-network-operator components +images: + operator: ghcr.io/k8snetworkplumbingwg/sriov-network-operator + sriovConfigDaemon: ghcr.io/k8snetworkplumbingwg/sriov-network-operator-config-daemon + sriovCni: ghcr.io/k8snetworkplumbingwg/sriov-cni + ibSriovCni: ghcr.io/k8snetworkplumbingwg/ib-sriov-cni + ovsCni: ghcr.io/k8snetworkplumbingwg/ovs-cni-plugin + rdmaCni: ghcr.io/k8snetworkplumbingwg/rdma-cni + sriovDevicePlugin: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin + sriovDraDriver: ghcr.io/k8snetworkplumbingwg/dra-driver-sriov:latest + resourcesInjector: ghcr.io/k8snetworkplumbingwg/network-resources-injector + webhook: ghcr.io/k8snetworkplumbingwg/sriov-network-operator-webhook + metricsExporter: ghcr.io/k8snetworkplumbingwg/sriov-network-metrics-exporter + metricsExporterKubeRbacProxy: quay.io/brancz/kube-rbac-proxy:v0.21.2 + +# DRA Driver configuration (used when dynamicResourceAllocation feature gate is enabled) +draDriver: + # CDI root directory for device specification + cdiRoot: "/var/run/cdi" + # Default interface prefix for VF naming + defaultInterfacePrefix: "net" + +imagePullSecrets: [] +extraDeploy: [] diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml new file mode 100644 index 00000000..8a201698 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml @@ -0,0 +1,58 @@ +# Copyright 2020 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - nad + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this represen + tation of an object. Servers should convert recognized schemas to the + latest internal value, and may reject unrecognized values. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_hostdevicenetworks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_hostdevicenetworks.yaml new file mode 100644 index 00000000..1d3bdd5a --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_hostdevicenetworks.yaml @@ -0,0 +1,111 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: hostdevicenetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: HostDeviceNetwork + listKind: HostDeviceNetworkList + plural: hostdevicenetworks + singular: hostdevicenetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: HostDeviceNetwork is the Schema for the hostdevicenetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of HostDeviceNetwork + properties: + ipam: + description: IPAM configuration to be used for this network + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: Host device resource pool name + type: string + type: object + status: + description: Defines the observed state of HostDeviceNetwork + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + message: + description: |- + Message is a human readable message indicating details about why + the state is in this condition + type: string + name: + description: Name of the deployed component this state refers + to + type: string + state: + description: The state of the deployed component. ("ready", + "notReady", "ignore", "error") + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + hostDeviceNetworkAttachmentDef: + description: Network attachment definition generated from HostDeviceNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the HostDeviceNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_ipoibnetworks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_ipoibnetworks.yaml new file mode 100644 index 00000000..95322d58 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_ipoibnetworks.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: ipoibnetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: IPoIBNetwork + listKind: IPoIBNetworkList + plural: ipoibnetworks + singular: ipoibnetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: IPoIBNetwork is the Schema for the ipoibnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of IPoIBNetwork + properties: + ipam: + description: IPAM configuration to be used for this network. + type: string + master: + description: Name of the host interface to enslave. Defaults to default + route interface + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + type: object + status: + description: Defines the observed state of IPoIBNetwork + properties: + ipoibNetworkAttachmentDef: + description: Network attachment definition generated from IPoIBNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the IPoIBNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_macvlannetworks.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_macvlannetworks.yaml new file mode 100644 index 00000000..ea0a31e7 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_macvlannetworks.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: macvlannetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: MacvlanNetwork + listKind: MacvlanNetworkList + plural: macvlannetworks + singular: macvlannetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: MacvlanNetwork is the Schema for the macvlannetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of MacvlanNetworkSpec + properties: + ipam: + description: IPAM configuration to be used for this network. + type: string + master: + description: Name of the host interface to enslave. Defaults to default + route interface + type: string + mode: + description: Mode of interface one of "bridge", "private", "vepa", + "passthru" + enum: + - bridge + - private + - vepa + - passthru + type: string + mtu: + description: MTU of interface to the specified value. 0 for master's + MTU + minimum: 0 + type: integer + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + type: object + status: + description: Defines the observed state of MacvlanNetwork + properties: + macvlanNetworkAttachmentDef: + description: Network attachment definition generated from MacvlanNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the MacvlanNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicclusterpolicies.yaml new file mode 100644 index 00000000..079eefe0 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicclusterpolicies.yaml @@ -0,0 +1,2187 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: nicclusterpolicies.mellanox.com +spec: + group: mellanox.com + names: + kind: NicClusterPolicy + listKind: NicClusterPolicyList + plural: nicclusterpolicies + shortNames: + - ncp + singular: nicclusterpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NicClusterPolicy is the Schema for the nicclusterpolicies API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of NicClusterPolicy + properties: + deploymentNodeAffinity: + description: NodeAffinity rules to inject to the Deployments objects + that are managed by the operator + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + deploymentTolerations: + description: Tolerations to inject to the Deployments objects that + are managed by the operator + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + docaTelemetryService: + description: |- + DOCATelemetryService exposes telemetry from NVIDIA networking components to prometheus. + See: https://docs.nvidia.com/doca/sdk/doca+telemetry+service+guide/index.html + properties: + config: + description: |- + Config contains custom config for the DOCATelemetryService. + If set no default config will be deployed. + properties: + fromConfigMap: + description: |- + FromConfigMap sets the configMap the DOCATelemetryService gets its configuration from. The ConfigMap must be in + the same namespace as the NICClusterPolicy. + type: string + type: object + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + global: + description: Global contains global configuration for all components + properties: + imagePullSecrets: + description: ImagePullSecrets is a list of secret names for pulling + component images + items: + type: string + type: array + repository: + description: Repository is the default container image repository + for all components + type: string + version: + description: Version is the default version tag for all component + images + type: string + type: object + ibKubernetes: + description: |- + IBKubernetes provides a daemon that works in conjunction with the SR-IOV Network Device Plugin. + It acts on Kubernetes pod object changes and reads the pod's network annotation. + From there it fetches the corresponding network CRD and reads the PKey. + This is done in order to add the newly generated GUID or the predefined GUID in the GUID field of the CRD. + This is then passed in cni-args to that PKey for pods with mellanox.infiniband.app annotation. + See: https://github.com/Mellanox/ib-kubernetes + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + pKeyGUIDPoolRangeEnd: + description: The last guid in the pool + type: string + pKeyGUIDPoolRangeStart: + description: The first guid in the pool + type: string + periodicUpdateSeconds: + default: 5 + description: Interval of updates in seconds + minimum: 0 + type: integer + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + ufmSecret: + description: Secret containing credentials to UFM service + type: string + version: + description: Version of the image to use + type: string + required: + - image + - pKeyGUIDPoolRangeEnd + - pKeyGUIDPoolRangeStart + - ufmSecret + type: object + nicConfigurationOperator: + description: |- + NicConfigurationOperator provides Kubernetes CRD API to allow FW configuration on NVIDIA NICs in a coordinated manner + See: https://github.com/Mellanox/nic-configuration-operator + properties: + configurationDaemon: + description: Image information for nic-configuration-daemon + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + env: + description: List of environment variables to set in the NIC Configuration + Operator and NIC Configuration Daemon containers. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + logLevel: + default: info + description: LogLevel sets the verbosity level of the logs. info|debug + enum: + - info + - debug + type: string + nicFirmwareStorage: + description: NicFirmwareStorage contains configuration for the + NIC firmware storage. If not provided, the NIC firmware storage + will not be configured. + properties: + availableStorageSize: + default: 1Gi + description: 'AvailableStorageSize is storage size for the + NIC Configuration Operator to request. Only applies if nicFirmwareStorage.create + == true. Default value: 1Gi' + pattern: ^(\d+)(Ei|Pi|Ti|Gi|Mi|Ki)$ + type: string + x-kubernetes-validations: + - message: availableStorageSize is immutable once set. nicFirmwareStorage + should be deleted and created again with a new value. + rule: self == oldSelf + create: + default: true + description: |- + Create specifies whether to create a new PVC or use an existing one + If create == false, the existing PVC with the name specified in pvcName should be located in the same namespace as the operator + type: boolean + pvcName: + default: nic-fw-storage-pvc + description: 'PVCName is the name of the PVC to mount as NIC + Firmware storage. Default value: "nic-fw-storage-pvc"' + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + storageClassName: + description: |- + StorageClassName is the name of a storage class to be used to store NIC FW binaries during NIC FW upgrade. + If not provided, the cluster-default storage class will be used + maxLength: 63 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + x-kubernetes-validations: + - message: storageClassName is immutable once set. nicFirmwareStorage + should be deleted and created again with a new value. + rule: self == oldSelf + type: object + operator: + description: Image information for nic-configuration-operator + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + required: + - configurationDaemon + - operator + type: object + nicFeatureDiscovery: + description: |- + NicFeatureDiscovery works with NodeFeatureDiscovery to expose information about NVIDIA NICs. + https://github.com/Mellanox/nic-feature-discovery + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + nodeAffinity: + description: NodeAffinity rules to inject to the DaemonSets objects + that are managed by the operator + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nvIpam: + description: |- + NvIpam is an IPAM provider that dynamically assigns IP addresses with speed and performance in mind. + Note: NvIPam requires certificate management e.g. cert-manager or OpenShift cert management. + See https://github.com/Mellanox/nvidia-k8s-ipam + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + enableWebhook: + description: Enable deployment of the validation webhook + type: boolean + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + ofedDriver: + description: |- + OFEDDriver is a specialized driver for NVIDIA NICs which can replace the inbox driver that comes with an OS. + See https://network.nvidia.com/support/mlnx-ofed-matrix/ + properties: + certConfig: + description: 'Optional: Custom TLS certificates configuration + for DOCA-OFED driver container' + properties: + name: + description: Name of the ConfigMap + type: string + type: object + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + env: + description: List of environment variables to set in the DOCA-OFED + driver container. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + forcePrecompiled: + default: false + description: |- + ForcePrecompiled specifies if only DOCA-OFED driver precompiled images are allowed + If set to false and precompiled image does not exists, DOCA-OFED driver will be compiled on Nodes + If set to true and precompiled image does not exists, OFED state will be Error. + type: boolean + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + livenessProbe: + description: Pod liveness probe settings + properties: + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + readinessProbe: + description: Pod readiness probe settings + properties: + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + repoConfig: + description: 'Optional: Custom package repository configuration + for DOCA-OFED driver container' + properties: + name: + description: Name of the ConfigMap + type: string + type: object + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + startupProbe: + description: Pod startup probe settings + properties: + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + terminationGracePeriodSeconds: + default: 300 + description: |- + TerminationGracePeriodSeconds specifies the length of time in seconds + to wait before killing the DOCA-OFED driver container pod on termination + format: int64 + minimum: 0 + type: integer + upgradePolicy: + description: DOCA-OFED driver auto-upgrade settings + properties: + autoUpgrade: + default: false + description: |- + AutoUpgrade is a global switch for automatic upgrade feature + if set to false all other options are ignored + type: boolean + drain: + description: The configuration for node drain during automatic + upgrade + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the node is drained) + type: boolean + enable: + default: true + description: Enable indicates if node draining is allowed + during upgrade + type: boolean + force: + default: false + description: Force indicates if force draining is allowed + type: boolean + podSelector: + description: |- + PodSelector specifies a label selector to filter pods on the node that need to be drained + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 300 + description: TimeoutSecond specifies the length of time + in seconds to wait before giving up drain, zero means + infinite + minimum: 0 + type: integer + type: object + maxParallelUpgrades: + default: 1 + description: |- + MaxParallelUpgrades indicates how many nodes can be upgraded in parallel + 0 means no limit, all nodes will be upgraded in parallel + minimum: 0 + type: integer + safeLoad: + default: false + description: SafeLoad turn on safe driver loading (cordon + and drain the node before loading the driver) + type: boolean + waitForCompletion: + description: The configuration for waiting on pods completions + properties: + podSelector: + description: |- + PodSelector specifies a label selector for the pods to wait for completion + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 0 + description: |- + TimeoutSecond specifies the length of time in seconds + to wait before giving up on pod termination, zero means infinite + minimum: 0 + type: integer + type: object + type: object + version: + description: Version of the image to use + type: string + required: + - image + type: object + rdmaSharedDevicePlugin: + description: |- + RdmaSharedDevicePlugin manages support IB and RoCE HCAs through the Kubernetes device plugin framework. + The config field is a json representation of the RDMA shared device plugin configuration. + See https://github.com/Mellanox/k8s-rdma-shared-dev-plugin + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + useCdi: + description: |- + Enables use of container device interface (CDI) + NOTE: NVIDIA Network Operator does not configure container runtime to enable CDI. + type: boolean + version: + description: Version of the image to use + type: string + required: + - image + type: object + secondaryNetwork: + description: |- + SecondaryNetwork Specifies components to deploy in order to facilitate a secondary network in Kubernetes. + It consists of the following optionally deployed components: + - Multus-CNI: Delegate CNI plugin to support secondary networks in Kubernetes + - CNI plugins: Currently only containernetworking-plugins is supported + - IPoIB CNI: Allows the user to create IPoIB child link and move it to the pod + properties: + cniPlugins: + description: Image information for CNI plugins + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + ipoib: + description: Image information for IPoIB CNI + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + multus: + description: Image and configuration information for multus + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + type: object + spectrumXOperator: + description: |- + SpectrumXOperator exposes NVIDIA Spectrum-X Operator. + See: https://github.com/Mellanox/spectrum-x-operator/ + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + xPlane: + description: ImageSpec Contains container image specifications + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + type: object + required: + - image + type: object + sriovDevicePlugin: + description: |- + SriovDevicePlugin manages SRIOV through the Kubernetes device plugin framework. + The config field is a json representation of the RDMA shared device plugin configuration. + See https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + useCdi: + description: |- + Enables use of container device interface (CDI) + NOTE: NVIDIA Network Operator does not configure container runtime to enable CDI. + type: boolean + version: + description: Version of the image to use + type: string + required: + - image + type: object + tolerations: + description: Tolerations to inject to the DaemonSets objects that + are managed by the operator + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + status: + description: Defines the observed state of NicClusterPolicy + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + message: + description: |- + Message is a human readable message indicating details about why + the state is in this condition + type: string + name: + description: Name of the deployed component this state refers + to + type: string + state: + description: The state of the deployed component. ("ready", + "notReady", "ignore", "error") + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + conditions: + description: |- + Conditions is a list of conditions describing the state of the NicClusterPolicy. + Each enabled component exposes a Ready condition, and the aggregate + Ready condition summarizes the overall policy health. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the current state of the cluster policy + enum: + - ignore + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicnodepolicies.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicnodepolicies.yaml new file mode 100644 index 00000000..f6078c14 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/crds/mellanox.com_nicnodepolicies.yaml @@ -0,0 +1,772 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: nicnodepolicies.mellanox.com +spec: + group: mellanox.com + names: + kind: NicNodePolicy + listKind: NicNodePolicyList + plural: nicnodepolicies + shortNames: + - nicnode + - nnp + singular: nicnodepolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NicNodePolicy is the Schema for the NicNodePolicies API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of NicNodePolicy + properties: + annotations: + additionalProperties: + type: string + description: |- + Optional: Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + type: object + labels: + additionalProperties: + type: string + description: |- + Optional: Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector specifies a selector for the nodes this + policy applies to + type: object + ofedDriver: + description: |- + OFEDDriver is a specialized driver for NVIDIA NICs which can replace the inbox driver that comes with an OS. + See https://network.nvidia.com/support/mlnx-ofed-matrix/ + properties: + certConfig: + description: 'Optional: Custom TLS certificates configuration + for DOCA-OFED driver container' + properties: + name: + description: Name of the ConfigMap + type: string + type: object + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + env: + description: List of environment variables to set in the DOCA-OFED + driver container. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + forcePrecompiled: + default: false + description: |- + ForcePrecompiled specifies if only DOCA-OFED driver precompiled images are allowed + If set to false and precompiled image does not exists, DOCA-OFED driver will be compiled on Nodes + If set to true and precompiled image does not exists, OFED state will be Error. + type: boolean + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + livenessProbe: + description: Pod liveness probe settings + properties: + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + readinessProbe: + description: Pod readiness probe settings + properties: + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + repoConfig: + description: 'Optional: Custom package repository configuration + for DOCA-OFED driver container' + properties: + name: + description: Name of the ConfigMap + type: string + type: object + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + startupProbe: + description: Pod startup probe settings + properties: + failureThreshold: + description: Minimum consecutive failures for the probe to + be considered failed after having succeeded + type: integer + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times + out + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + terminationGracePeriodSeconds: + default: 300 + description: |- + TerminationGracePeriodSeconds specifies the length of time in seconds + to wait before killing the DOCA-OFED driver container pod on termination + format: int64 + minimum: 0 + type: integer + upgradePolicy: + description: DOCA-OFED driver auto-upgrade settings + properties: + autoUpgrade: + default: false + description: |- + AutoUpgrade is a global switch for automatic upgrade feature + if set to false all other options are ignored + type: boolean + drain: + description: The configuration for node drain during automatic + upgrade + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the node is drained) + type: boolean + enable: + default: true + description: Enable indicates if node draining is allowed + during upgrade + type: boolean + force: + default: false + description: Force indicates if force draining is allowed + type: boolean + podSelector: + description: |- + PodSelector specifies a label selector to filter pods on the node that need to be drained + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 300 + description: TimeoutSecond specifies the length of time + in seconds to wait before giving up drain, zero means + infinite + minimum: 0 + type: integer + type: object + maxParallelUpgrades: + default: 1 + description: |- + MaxParallelUpgrades indicates how many nodes can be upgraded in parallel + 0 means no limit, all nodes will be upgraded in parallel + minimum: 0 + type: integer + safeLoad: + default: false + description: SafeLoad turn on safe driver loading (cordon + and drain the node before loading the driver) + type: boolean + waitForCompletion: + description: The configuration for waiting on pods completions + properties: + podSelector: + description: |- + PodSelector specifies a label selector for the pods to wait for completion + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 0 + description: |- + TimeoutSecond specifies the length of time in seconds + to wait before giving up on pod termination, zero means infinite + minimum: 0 + type: integer + type: object + type: object + version: + description: Version of the image to use + type: string + required: + - image + type: object + rdmaSharedDevicePlugin: + description: |- + RdmaSharedDevicePlugin manages support IB and RoCE HCAs through the Kubernetes device plugin framework. + The config field is a json representation of the RDMA shared device plugin configuration. + See https://github.com/Mellanox/k8s-rdma-shared-dev-plugin + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + useCdi: + description: |- + Enables use of container device interface (CDI) + NOTE: NVIDIA Network Operator does not configure container runtime to enable CDI. + type: boolean + version: + description: Version of the image to use + type: string + required: + - image + type: object + sriovDevicePlugin: + description: |- + SriovDevicePlugin manages SRIOV through the Kubernetes device plugin framework. + The config field is a json representation of the RDMA shared device plugin configuration. + See https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + useCdi: + description: |- + Enables use of container device interface (CDI) + NOTE: NVIDIA Network Operator does not configure container runtime to enable CDI. + type: boolean + version: + description: Version of the image to use + type: string + required: + - image + type: object + tolerations: + description: 'Optional: Set tolerations' + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + status: + description: Defines the observed state of NicNodePolicy + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + message: + description: |- + Message is a human readable message indicating details about why + the state is in this condition + type: string + name: + description: Name of the deployed component this state refers + to + type: string + state: + description: The state of the deployed component. ("ready", + "notReady", "ignore", "error") + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + conditions: + description: |- + Conditions is a list of conditions describing the state of the NicNodePolicy. + Each enabled component exposes a Ready condition, and the aggregate + Ready condition summarizes the overall policy health. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the current state of the cluster policy + enum: + - ignore + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/NOTES.txt b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/NOTES.txt new file mode 100644 index 00000000..6f039352 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/NOTES.txt @@ -0,0 +1,3 @@ +Get Network Operator deployed resources by running the following commands: + +$ kubectl -n {{ .Release.Namespace }} get pods diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/_helpers.tpl b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/_helpers.tpl new file mode 100644 index 00000000..198e47ad --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/_helpers.tpl @@ -0,0 +1,86 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "network-operator.name" -}} +{{- default .Chart.Name .Values.operator.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "network-operator.fullname" -}} +{{- if .Values.operator.fullnameOverride }} +{{- .Values.operator.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.operator.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "network-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "network-operator.labels" -}} +helm.sh/chart: {{ include "network-operator.chart" . }} +{{ include "network-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "network-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "network-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +imagePullSecrets helpers +*/}} +{{- define "network-operator.operator.imagePullSecrets" }} +{{- $imagePullSecrets := list }} +{{- if .Values.operator.imagePullSecrets }} +{{- range .Values.operator.imagePullSecrets }} +{{- $imagePullSecrets = append $imagePullSecrets (dict "name" . ) }} +{{- end }} +{{- else }} +{{- if .Values.imagePullSecrets }} +{{- range .Values.imagePullSecrets }} +{{- $imagePullSecrets = append $imagePullSecrets (dict "name" . ) }} +{{- end }} +{{- end }} +{{- end }} +{{- $imagePullSecrets | toJson }} +{{- end }} \ No newline at end of file diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/admission_controller.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/admission_controller.yaml new file mode 100644 index 00000000..1bfa736f --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/admission_controller.yaml @@ -0,0 +1,152 @@ +--- +{{- if .Values.operator.admissionController.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: {{ .Release.Name }}-controller +{{- end }} +--- +{{- if and .Values.operator.admissionController.enabled .Values.operator.admissionController.useCertManager }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: {{ .Release.Name }} + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: {{ .Release.Name }} + name: {{ .Release.Name }}-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - {{ .Release.Name }}-webhook-service.{{ .Release.Namespace }}.svc + - {{ .Release.Name }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ .Release.Name }}-selfsigned-issuer + secretName: webhook-server-cert +{{- end }} +--- +{{- if and .Values.operator.admissionController.enabled .Values.operator.admissionController.useCertManager }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: {{ .Release.Name }} + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: {{ .Release.Name }} + name: {{ .Release.Name }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- end }} +--- +{{- if .Values.operator.admissionController.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + {{- if .Values.operator.admissionController.useCertManager }} + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-serving-cert + {{- end }} + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: {{ .Release.Name }} + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/part-of: {{ .Release.Name }} + name: {{ .Release.Name }}-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-mellanox-com-v1alpha1-hostdevicenetwork + {{- if not .Values.operator.admissionController.useCertManager }} + caBundle: {{ .Values.operator.admissionController.certificate.caCrt | b64enc | quote }} + {{- end }} + failurePolicy: Fail + name: vhostdevicenetwork.kb.io + rules: + - apiGroups: + - mellanox.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - hostdevicenetworks + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-mellanox-com-v1alpha1-nicclusterpolicy + {{- if not .Values.operator.admissionController.useCertManager }} + caBundle: {{ .Values.operator.admissionController.certificate.caCrt | b64enc | quote }} + {{- end }} + failurePolicy: Fail + name: vnicclusterpolicy.kb.io + rules: + - apiGroups: + - mellanox.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - nicclusterpolicies + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-mellanox-com-v1alpha1-nicnodepolicy + {{- if not .Values.operator.admissionController.useCertManager }} + caBundle: {{ .Values.operator.admissionController.certificate.caCrt | b64enc | quote }} + {{- end }} + failurePolicy: Fail + name: vnicnodepolicy.kb.io + rules: + - apiGroups: + - mellanox.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - nicnodepolicies + sideEffects: None +{{- end }} +--- +{{- if and .Values.operator.admissionController.enabled (not .Values.operator.admissionController.useCertManager) }} +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: {{ .Release.Namespace }} +type: Opaque +data: + tls.crt: {{ .Values.operator.admissionController.certificate.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionController.certificate.tlsKey | b64enc | quote }} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/keep-ncp-hook.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/keep-ncp-hook.yaml new file mode 100644 index 00000000..edeeb69d --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/keep-ncp-hook.yaml @@ -0,0 +1,94 @@ +{{- if .Values.keepNCP }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-sa + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-role + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - mellanox.com + resources: + - nicclusterpolicies + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-scale-binding + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: network-operator-keep-ncp + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + {{- include "network-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "network-operator" +spec: + template: + metadata: + name: network-operator-keep-ncp + labels: + {{- include "network-operator.labels" . | nindent 8 }} + app.kubernetes.io/component: "network-operator" + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity}} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-sa + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }} + containers: + - name: keep-ncp + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + drop: + - ALL + runAsNonRoot: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + command: + - /keep-ncp + restartPolicy: OnFailure +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/nodefeaturerules.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/nodefeaturerules.yaml new file mode 100644 index 00000000..0dc04f84 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/nodefeaturerules.yaml @@ -0,0 +1,16 @@ +{{- if .Values.nfd.deployNodeFeatureRules }} +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: nvidia-nics-rules +spec: + rules: + - name: "Nvidia NICs PCI" + labels: + "pci-15b3.present": "true" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["15b3"]} + class: {op: In, value: ["0200", "0207"]} +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/operator.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/operator.yaml new file mode 100644 index 00000000..d07fcca8 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/operator.yaml @@ -0,0 +1,179 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "network-operator.fullname" . }} + labels: + {{- include "network-operator.labels" . | nindent 4 }} + control-plane: {{ .Release.Name }}-controller + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "network-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + labels: + nvidia.com/ofed-driver-upgrade-drain.skip: "true" + control-plane: {{ .Release.Name }}-controller + {{- include "network-operator.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity}} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + serviceAccountName: {{ include "network-operator.fullname" . }} + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }} + containers: + - name: {{ .Chart.Name }} + # Replace this with the built image name + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + {{- if .Values.operator.admissionController.enabled }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + {{- end }} + {{- if or .Values.operator.coverage.enabled .Values.operator.admissionController.enabled }} + volumeMounts: + {{- if .Values.operator.coverage.enabled }} + - name: go-coverage + mountPath: /coverage + {{- end }} + {{- if .Values.operator.admissionController.enabled }} + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + {{- end }} + {{- if gt (int (.Values.operator.preStopSleepSeconds | default 0)) 0 }} + lifecycle: + preStop: + sleep: + seconds: {{ .Values.operator.preStopSleepSeconds }} + {{- end }} + command: + - /manager + args: + - --leader-elect + env: + - name: STATE_MANIFEST_BASE_DIR + value: "/manifests" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_NAME + value: "network-operator" + {{- if .Values.operator.coverage.enabled }} + - name: GOCOVERDIR + value: "/coverage" + - name: COVERAGE_CLEAR_AFTER_FLUSH + value: "1" + {{- end }} + - name: ENABLE_WEBHOOKS + value: "{{ .Values.operator.admissionController.enabled }}" + - name: USE_DTK + value: "{{ .Values.operator.useDTK }}" + {{- if .Values.operator.cniBinDirectory }} + - name: CNI_BIN_DIR + value: "{{ .Values.operator.cniBinDirectory }}" + {{- end }} + {{- if .Values.operator.cniNetworkDirectory }} + - name: CNI_NETWORK_DIR + value: "{{ .Values.operator.cniNetworkDirectory }}" + {{- end }} + {{- if and .Values.operator.ofedDriver.initContainer .Values.operator.ofedDriver.initContainer.enable }} + - name: OFED_INIT_CONTAINER_IMAGE + {{- with .Values.operator.ofedDriver.initContainer }} + value: "{{ .repository }}/{{ .image }}:{{ .version }}" + {{- end }} + {{- end }} + {{- if .Values.operator.maintenanceOperator.useRequestor }} + - name: MAINTENANCE_OPERATOR_ENABLED + value: "{{ .Values.operator.maintenanceOperator.useRequestor }}" + - name: MAINTENANCE_OPERATOR_REQUESTOR_ID + value: {{ .Values.operator.maintenanceOperator.requestorID }} + - name: MAINTENANCE_OPERATOR_NODE_MAINTENANCE_PREFIX + value: {{ .Values.operator.maintenanceOperator.nodeMaintenanceNamePrefix }} + - name: MAINTENANCE_OPERATOR_REQUESTOR_NAMESPACE + value: {{ .Values.operator.maintenanceOperator.nodeMaintenanceNamespace }} + {{- end }} + {{- if .Values.operator.maintenanceOperator.useDrainControllerRequestor }} + - name: DRAIN_CONTROLLER_ENABLED + value: "{{ .Values.operator.maintenanceOperator.useDrainControllerRequestor }}" + - name: DRAIN_CONTROLLER_REQUESTOR_ID + value: {{ .Values.operator.maintenanceOperator.drainControllerRequestorID }} + - name: DRAIN_CONTROLLER_REQUESTOR_NAMESPACE + value: {{ .Values.operator.maintenanceOperator.nodeMaintenanceNamespace }} + - name: DRAIN_CONTROLLER_SRIOV_NODE_STATE_NAMESPACE + value: {{ .Release.Namespace }} + {{- end }} + securityContext: + capabilities: + drop: + - ALL + runAsNonRoot: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + {{- with .Values.operator.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + runAsUser: 65532 + terminationGracePeriodSeconds: {{ .Values.operator.terminationGracePeriodSeconds | default 30 }} + {{- if or .Values.operator.coverage.enabled .Values.operator.admissionController.enabled }} + volumes: + {{- if .Values.operator.coverage.enabled }} + - name: go-coverage + emptyDir: {} + {{- end }} + {{- if .Values.operator.admissionController.enabled }} + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert + {{- end }} + {{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role.yaml new file mode 100644 index 00000000..6d2064ac --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role.yaml @@ -0,0 +1,389 @@ +{{/* + Copyright 2025 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + // Code generated by controller-gen. DO NOT EDIT. +*/}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: {{ include "network-operator.fullname" . }} + +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - events + - namespaces + - persistentvolumeclaims + - pods + - pods/eviction + - pods/finalizers + - pods/status + - secrets + - serviceaccounts + - services + - services/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - configmaps/finalizers + verbs: + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments/finalizers + verbs: + - update +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + - issuers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - config.openshift.io + resources: + - clusterversions + - proxies + verbs: + - get + - list + - watch +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicconfigurationtemplates + - nicdevices + - nicfirmwaresources + - nicfirmwaretemplates + - nicinterfacenametemplates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicconfigurationtemplates/finalizers + - nicdevices/finalizers + - nicfirmwaresources/finalizers + - nicfirmwaretemplates/finalizers + verbs: + - update +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicconfigurationtemplates/status + - nicdevices/status + - nicfirmwaresources/status + - nicfirmwaretemplates/status + - nicinterfacenametemplates/status + verbs: + - get + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - get + - patch + - update +- apiGroups: + - image.openshift.io + resources: + - imagestreams + verbs: + - get + - list + - watch +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - maintenance.nvidia.com + resources: + - nodemaintenances + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - maintenance.nvidia.com + resources: + - nodemaintenances/status + verbs: + - get + - list + - watch +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks + - ipoibnetworks + - macvlannetworks + - nicclusterpolicies + - nicclusterpolicies/status + - nicnodepolicies + - nicnodepolicies/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks/finalizers + - ipoibnetworks/finalizers + - macvlannetworks/finalizers + - nicclusterpolicies/finalizers + - nicnodepolicies/finalizers + verbs: + - update +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks/status + - ipoibnetworks/status + - macvlannetworks/status + verbs: + - get + - patch + - update +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - get + - list + - watch +- apiGroups: + - nv-ipam.nvidia.com + resources: + - cidrpools + - ippools + verbs: + - create + - deletecollection + - get + - list + - watch +- apiGroups: + - nv-ipam.nvidia.com + resources: + - cidrpools/status + - ippools/status + verbs: + - get + - patch + - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + - resourceclaims/status + - resourceslices + verbs: + - get + - list +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use +- apiGroups: + - spectrumx.nvidia.com + resources: + - spectrumxrailpoolconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - spectrumx.nvidia.com + resources: + - spectrumxrailpoolconfigs/finalizers + verbs: + - update +- apiGroups: + - spectrumx.nvidia.com + resources: + - spectrumxrailpoolconfigs/status + verbs: + - get + - patch + - update +- apiGroups: + - sriovnetwork.openshift.io + resources: + - ovsnetworks + - sriovnetworknodepolicies + - sriovnetworkpoolconfigs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - sriovnetwork.openshift.io + resources: + - sriovnetworknodestates + verbs: + - get + - list + - patch + - watch diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role_binding.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role_binding.yaml new file mode 100644 index 00000000..9f4d698c --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/role_binding.yaml @@ -0,0 +1,27 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "network-operator.fullname" . }} +subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:serviceaccounts:{{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "network-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/service_account.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/service_account.yaml new file mode 100644 index 00000000..b98dc38e --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/service_account.yaml @@ -0,0 +1,20 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/upgrade-crd-hook.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/upgrade-crd-hook.yaml new file mode 100644 index 00000000..da29a94e --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/templates/upgrade-crd-hook.yaml @@ -0,0 +1,105 @@ +{{- if .Values.upgradeCRDs }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-sa + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-role + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-binding + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: {{ include "network-operator.fullname" . }}-hooks-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "network-operator.fullname" . }}-hooks-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: network-operator-upgrade-crd + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + {{- include "network-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "network-operator" +spec: + template: + metadata: + name: network-operator-upgrade-crd + labels: + {{- include "network-operator.labels" . | nindent 8 }} + app.kubernetes.io/component: "network-operator" + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity}} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "network-operator.fullname" . }}-hooks-sa + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }} + containers: + - name: upgrade-crd + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + drop: + - ALL + runAsNonRoot: true + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + command: + - /apply-crds + args: + - --crds-dir=/crds/network-operator + {{- if .Values.sriovNetworkOperator.enabled }} + - --crds-dir=/crds/sriov-network-operator + {{- end }} + {{- if .Values.nfd.enabled }} + - --crds-dir=/crds/node-feature-discovery + {{- end }} + {{- if .Values.maintenanceOperator.enabled }} + - --crds-dir=/crds/maintenance-operator + {{- end }} + restartPolicy: OnFailure +{{- end }} diff --git a/packs/nvidia-network-operator-26.7.0/charts/network-operator/values.yaml b/packs/nvidia-network-operator-26.7.0/charts/network-operator/values.yaml new file mode 100644 index 00000000..dbd964d4 --- /dev/null +++ b/packs/nvidia-network-operator-26.7.0/charts/network-operator/values.yaml @@ -0,0 +1,349 @@ +# Copyright 2020 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for network-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +nfd: + # -- Deploy Node Feature Discovery operator. + enabled: true + # -- Deploy Node Feature Rules to label the nodes with the discovered features. + deployNodeFeatureRules: true + +# -- Enable CRDs upgrade with helm pre-install and pre-upgrade hooks. +upgradeCRDs: true + +# -- Preserve a Helm-managed NicClusterPolicy custom resource during chart upgrades. +keepNCP: true + +sriovNetworkOperator: + # -- Deploy SR-IOV Network Operator. + enabled: false + +maintenanceOperator: + # -- Deploy Maintenance Operator. + enabled: false + +# Set both enableNodeFeatureApi and NodeFeatureAPI feature gate to false to disable. +node-feature-discovery: + # -- The Node Feature API enable communication between nfd master and worker + # through NodeFeature CRs. Otherwise communication is through gRPC. + enableNodeFeatureApi: true + featureGates: + NodeFeatureAPI: true + # -- Enable labels cleanup when uninstalling NFD + postDeleteCleanup: false + + image: + repository: nvcr.io/nvidia/mellanox/node-feature-discovery + tag: network-operator-v26.7.0 + pullPolicy: IfNotPresent + # imagePullSecrest for node-feature-discovery Network Operator related images + #imagePullSecrets: [] + + # -- NFD master deployment configuration. + # @notationType -- yaml + master: + serviceAccount: + name: node-feature-discovery + create: true + config: + extraLabelNs: ["nvidia.com"] + gc: + # -- Specifies whether the NFD Garbage Collector should be created + enable: true + # -- Specifies the number of replicas for the NFD Garbage Collector + replicaCount: 1 + serviceAccount: + # -- The name of the service account for garbage collector to use. + # If not set and create is true, a name is generated using the fullname + # template and -gc suffix. + name: node-feature-discovery + # -- disable creation to avoid duplicate serviceaccount creation by master + # spec above. + create: false + # -- NFD worker daemonset configuration. + # @notationType -- yaml + worker: + serviceAccount: + # disable creation to avoid duplicate serviceaccount creation by master spec + # above + name: node-feature-discovery + create: false + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + config: + sources: + pci: + deviceClassWhitelist: + - "0300" + - "0302" + deviceLabelFields: + - vendor + +# SR-IOV Network Operator chart related values. +sriov-network-operator: + operator: + # -- Prefix to be used for resources names. + resourcePrefix: "nvidia.com" + # -- Enable admission controller. + # @notationType -- yaml + admissionControllers: + enabled: false + certificates: + secretNames: + operator: "operator-webhook-cert" + injector: "network-resources-injector-cert" + certManager: + # -- When enabled, makes use of certificates managed by cert-manager. + enabled: true + # -- When enabled, certificates are generated via cert-manager and then + # name will match the name of the secrets defined above. + generateSelfSigned: true + # -- If not specified, no secret is created and secrets with the names + # defined above are expected to exist in the cluster. In that case, + # the ca.crt must be base64 encoded twice since it ends up being an env variable. + custom: + enabled: false + # operator: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + # injector: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + + # Image URIs for sriov-network-operator components. + images: + operator: nvcr.io/nvidia/mellanox/sriov-network-operator:network-operator-v26.7.0 + sriovConfigDaemon: nvcr.io/nvidia/mellanox/sriov-network-operator-config-daemon:network-operator-v26.7.0 + sriovCni: nvcr.io/nvidia/mellanox/sriov-cni:network-operator-v26.7.0 + ibSriovCni: nvcr.io/nvidia/mellanox/ib-sriov-cni:network-operator-v26.7.0 + ovsCni: nvcr.io/nvidia/mellanox/ovs-cni-plugin:network-operator-v26.7.0 + rdmaCni: nvcr.io/nvidia/mellanox/rdma-cni:network-operator-v26.7.0 + sriovDevicePlugin: nvcr.io/nvidia/mellanox/sriov-network-device-plugin:network-operator-v26.7.0 + sriovDraDriver: nvcr.io/nvidia/mellanox/dra-driver-sriov:network-operator-v26.7.0 + resourcesInjector: ghcr.io/k8snetworkplumbingwg/network-resources-injector:v1.7.0 + webhook: nvcr.io/nvidia/mellanox/sriov-network-operator-webhook:network-operator-v26.7.0 + # imagePullSecrest for SR-IOV Network Operator related images + # imagePullSecrets: [] + sriovOperatorConfig: + # -- Deploy ``SriovOperatorConfig`` custom resource + deploy: true + # -- Selects the nodes to be configured + # @notationType -- yaml + configDaemonNodeSelector: + beta.kubernetes.io/os: "linux" + network.nvidia.com/operator.mofed.wait: "false" + # Enable when using together with NIC Configuration Operator to wait until + # all required FW parameters are successfully applied before configuring SR-IOV + # network.nvidia.com/operator.nic-configuration.wait: "false" + +# Maintenance Operator chart related values. +maintenance-operator-chart: + # -- Deploy MaintenanceOperatorConfig. + # Maintenance Operator might be already deployed on the cluster, in that case no need to deploy MaintenanceOperatorConfig. + operatorConfig: + # -- deploy MaintenanceOperatorConfig CR + deploy: false + # imagePullSecrets for Maintenance Operator related images + #imagePullSecrets: [] + operator: + image: + repository: nvcr.io/nvidia/mellanox + name: maintenance-operator + tag: network-operator-v26.7.0 + admissionController: + # -- enable admission controller of the operator + enable: false + certificates: + secretNames: + # -- secret name containing certificates for the operator admission controller + operator: "maintenance-webhook-cert" + certManager: + # -- use cert-manager for certificates + enable: false + # -- generate self-signed certificates with cert-manager + generateSelfSigned: false + custom: + # -- enable custom certificates using secrets + enable: false + +# General Operator related values +# The operator element allows to deploy network operator from an alternate location +operator: + # -- Optional `resource requests and limits `_ + # for the operator. + # @notationType -- yaml + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + # -- Priority class for the operator controller. ``system-node-critical`` keeps the + # controller in the same graceful node shutdown phase as MOFED so it can update + # ``network.nvidia.com/operator.mofed.wait`` before the node goes down. See + # https://github.com/Mellanox/network-operator/issues/2954. + priorityClassName: system-node-critical + # -- Termination grace period for the operator controller. Must be greater than + # ``preStopSleepSeconds``. + terminationGracePeriodSeconds: 30 + # -- Seconds to sleep in ``preStop`` before SIGTERM. Keeps the controller reconciling + # while MOFED pods begin terminating during graceful node shutdown. Requires + # Kubernetes >= 1.29 (PodLifecycleSleepAction). Set to ``0`` to disable. + preStopSleepSeconds: 25 + # -- Set additional tolerations for various Daemonsets deployed by the operator. + # @notationType -- yaml + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "" + effect: "NoSchedule" + # -- Configure node selector settings for the operator. + nodeSelector: {} + affinity: + # -- Configure node affinity settings for the operator. + # @notationType -- yaml + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [ "" ] + # -- Network Operator image repository. + repository: nvcr.io/nvidia/cloud-native + # -- Network Operator image name + image: network-operator + # -- Enable Go integration coverage collection (GOCOVERDIR, coverage volume, instrumented image tag). + coverage: + enabled: false + # imagePullSecrets: [] + # -- Name to be used as part of objects name generation. + nameOverride: "" + # -- Name to be used to replace generated names. + fullnameOverride: "" + # tag, if defined will use the given image tag, else Chart. AppVersion will be used. + # tag + # -- Directory, where CNI binaries will be deployed on the nodes. Setting for + # the sriov-network-operator is set with ``sriov-network-operator.cniBinPath`` + # parameter. Note that the CNI bin directory should be aligned with the CNI bin + # directory in the container runtime. + cniBinDirectory: /opt/cni/bin + # -- Directory, where CNI network configuration will be deployed on the nodes. + # Note that the CNI network directory should be aligned with the CNI network + # directory in the container runtime. + cniNetworkDirectory: /etc/cni/net.d + # -- Enable the use of maintenance operator upgrade logic. + maintenanceOperator: + # k8s namespace to be used for generated nodeMaintenance objects + nodeMaintenanceNamespace: default + # enable upgrade controller requestor + useRequestor: false + # upgrade controller requestor ID to be used in nodeMaintenance objects + requestorID: "nvidia.doca-driver-upgrade" + # nodeMaintenance name prefix to be used by upgrade-controller (default is "") + nodeMaintenanceNamePrefix: "network-operator" + # enable drain controller requestor + useDrainControllerRequestor: false + # drain controller requestor ID to be used in nodeMaintenance objects + drainControllerRequestorID: "nvidia.network-operator-drain-controller" + # -- Enable the use of Driver ToolKit to compile DOCA-OFED Drivers (OpenShift only). + useDTK: true + admissionController: + # -- Deploy with admission controller. + enabled: false + # -- Use cert-manager for generating self-signed certificate. + useCertManager: true + # certificate: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + ofedDriver: + initContainer: + # -- Deploy init container. + enable: true + # -- Init container image repository. + repository: nvcr.io/nvidia/mellanox + # -- Init container image name. + image: network-operator-init-container + # -- Init container image version. + version: network-operator-v26.7.0 + +# -- An optional list of references to secrets to use for pulling any of the +# Network Operator images. +imagePullSecrets: [] + +# @ignore +test: + pf: ens2f0 diff --git a/packs/nvidia-network-operator-26.7.0/logo.png b/packs/nvidia-network-operator-26.7.0/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..a98c78690812d65f46914a8f9319c7ed2a32fefe GIT binary patch literal 7892 zcmc(EWmFv97G*a90u2PW0HK2f_XKIQgS!Ws;O^2i(l`wPfR0951EvjB6$${<#NpkTVcj2tEflm=0RW$806}><;l3XZRNba&P~2 z^Duz_ML{@-Gw7>ofTdmFR$xJHUT$6n2@n_z7K2+_L$qY%{uzFMB+g)qK)6D9c)Yy4 zxV;3pUEnr6d?F$uJiPoo{QO+^3@&%1Gs4W9%h{dr-y;80N5;zC0&eGuuyb(+|J5}! zckw`oGcf!O^q=t@Hsh8 z=lC&o&M#sWVnI@pQqRvkYB6=!8d@m%)*k97ex9X?f`47mvF~HjZSI~ie%t$NL9vKl zszAH^qFRae)o&|y!n|N((~io9maC^WNB&gLryl@gu(1WQaz{I)PX@O}W2&za=XTF| zmEMwKVgjL3L`45zfyy59Ykt*ygwF4GE@QPvV|&Eqzkj@+&e0Oh{a(pU@O23U%BqkG ziy*@6qyTEh*Ekt8*RT^2VU=UZCh#@lM4+ObMtfs)gM(NV$XO$!Yh(mgq zS>9)?yvGvNp^ctND&_VL-~O7RMLDZ*JwWxAxFLuRj;D1{PWGulKwX68oWYL1H>o^i0m(_ZchN5wUlayW0~h;u$=3 z)lFk#ch9}k+JjPUQM?JbX#jm(FhY?GJlzyfo9kf&gGid7samc&)f&`>8JAJ#IuHDq zv|(jMD{4f+Z}o_#NTCi&Mjep1O2k~*Y9GZ53L0wjoE<6$KD?AYRR3)JPCuX?p4Lut zhko5KbT&3Vg}sT3(LIG($@1o=%a>iHGnwG-%%K6TTCR&e@9PyV_bc?9^Ist=>WZak zoYluxEf=TSjw5{i@o7%Q?Py>rw*|IRAbq>`wSMKES<_DbMmc?65~0O)p=+Sgv}VJo z`gmkUXS5JgU3Q;p1nN^BqE)PzEF}9&apN$xmx-9}EKRD(Lg$)OvbPy~XuMogL;}8~ z!KZ@@ny;;kqV87r$JWXlKLd>y-`o{>%RtpYNag15cdvd)wAFUO_;_EJNqg*EYBwww zkXa9lvqw;no6AjQK_aY^HRRWgns*_mDwSrJo-g9gi$>&O(2u7Q0l(kvk8iuHi@Rz) zVwY2{l)O{?6>yegz)?XPFOB?~_Zv&C$7-YlUwaS zal!rX{)Ov=wwz70DYWAKR0EqS7Fn{5s;oWRI?6`ydED!j)a`|?#R_rt@-JYCbV@xS zhDAQuuiNX2yul~$W83PIsmZQN(-`^WKq?E#(acoGOxvCRK^VF#u{7-Bag_wBsKA+` zu7|NdY$vzn_J{(W@sVdiv)*Rm26IsTqv31S)r3#xIHn(|oz92Pw|}Lle2)5Z;J)9k ztA>Wn$mBCYVu`C=y(v#m#PHJatMk$AUDg5;LCtt2Qv~Hv=H06e z6Y-b^&sirX?8WuE@8u4%_y@MrXZ9yj&dk*zK;s07yaq}h zjc2kt^g&Y|e$$;83>GraH$oj7rV`;89pfzWVK5eh4YhtQcn57?cQwUt6>j+v5H3yd zi$sS)*~2s)(J|y!xTh@`nPFkyerYOB1e~{g@qemh9x|FENCHC5g9&GIgz%+ zhdHEHZ1ygBwFz*)KRGk~?}e?tdp`4Vu8r^NL$I8 z)6$tR>#)YF(|?td8p^~FaQuBL;sQ1=|6rU{vB%VmIr!zE1)F@EKiD~m+|o8x|5oRb z&Fip1Rnx8|)-J!du@a&k7K~W<_H|8vwsObb(VMvOSBZxC2CuT|_wJdBG&`rT{6S!b zy@Ce#F!R5hg@8fd%MpTaF-R0SvJHJgcWH5 zOO|zsOyihIV^+f~1ja75vN(;i0#{niIXH~Eno7q5I(qYEIkKjE=TnOPLrHs_yQEDS z&6+p;R`ilq%h5S7V*ZZ$eadGM;3A8VC;)6e%7R%c|ewJl2xs z{WUCM9uQ4eKwB>Lib`Y-grboxU1(>=qq~#pr)Zw~Gq+ z=}CL=G;tE>Z5~*$twju`!nS$c6btjgwkeWBKVCWW;B65@-oFruqJ6UN{4`hcxMF^D zFma!1zl{MtkMAnsr|644DDpe+QPo*=l{2Z_otPl*dd*zl{N8AK=PiUE)ao>oqD3z< zMd(;J2Sdks?+Lo#QDar<$NYhwJaSw!zw+zrSRXpW|Lst2I$qGkb=e{BfAZw~8$Pue zRK%i7f*SPYrybj>s$X{F%SI& zHs%(W1lB0ex>zENj7eh`Q0-Bc8YM9pXZUd%63ZkYf%XjL+ml+nNPbbfvz|>h9D^}8 zbli-DOS$aqdUbV~1Fy!pV>Z@Cc1n$<@6cd6W@B6I4zBqOjdxyi2!wJzwuCRBr=vu1 zC7&EiV7uL~!;OhNl%Okto{oLLTo~3o_SsC7b@S#omNf~Y_hF+I4Ibz61$yo=i?zy{GhZf)a7S5;bQ zD}Ge=WOI22gUQ+&iFWWO6M60o5)Asho(?PZ9V=pI%lxkld)-6q4u7aA{`6 zvVz0Bu5>wz4f{5$6i836^Cn@1uPs?*-gGDieZhg*)M^H-d7{ZfRdvyH4j*#&P2ai} zkG4D>Px^F7mkset>GnW9D_wv< zBCW}Jws>TXtVNCW>DciU70V@4^~lU5b?RPPH!0iXK@(_*gxt^p4$gxu6bUXnJgE&)1ON73BOx;Z6egm%DE%l zE6_c~hC@>nkr>o+ptp#Wz|-C**^LyrUl_7RyaG+JK&NB7d^>7uH*ygX=%Sm|pIz>y z2eQgmfs2yN4thzf3Y4^M#HWd;5pgPlaPASsakt`8=X|FpLo&0oCb}!S8qdAHvwnjb^o$4=r*hqM=idCx$tzJJaK;$9C> z)+>7#Sw5^&BxO*VW?}Q*knd0ezP7sVP`7`-3FoN>3{Zrp!tD;0jt^H9WPXA=Ee8ZnkmwjXsl=~ zSIA_V=DY4f=mFWCKB9*z(xsw?%_q|x3X3O?#*qlrc}-}a&!AY5*un8*bCF{zj*Nph zr~>Nm0xxqavYJC*BB8qQzHVYHwvZVj^l}w}TH%pUhcAEcFpxviUJE(dxN8uk>Gb3hHp0MyuVXwzT?+CE} zAYmR-U%G!fVCJSyU{E5^!*8?tr=^(8DjwARekB=H+TTJ?nm-g;cpJp({By!=*CZ&{ zivLkp<&q3rMC*_2{sCriX{#yI`5~haWyzy4&V{Mju7pr*nKxvVgEGUJK_7_=N=L#+ z^+9wpNc(#7_zeuQU_1H9A758D<)#%VvC8x0&rWEIP)FfL?SCI22{%*_IR4H3J0t}^ z2@ehcs7bVijS9lLM?B#~ru(-{1$rV(&U@7Vk-!9%B9pla0-Hjma*AI6a8qp^-DkP5 zm2OK89J%;PxxHL8UpH3`2IJsh+^hbd!4EzRmeXPEyHB8EKeKYfL`5PhAZA(8O|Lfj z$piFm(1aAQdtIL!>9qD|`WcwfNkabFx1-V1kmr3OH|N^DJ$)8mN?GDd-DZLUf$f6? z&5)G|OsnLd*LYG^h6&e7ZN)0-(z)S5Ho@}_H;moJ_27>w+JZE*56RseOdH3qjbG{; zmn!VwAd!x1HpTPMmK&M*5!4fg%sft!c1{4f2NM2nEt#v-w&n9h*WR0Q*xdZHi0uA{ z%=6Ct49xw+6XP`ri%qEyq`WOQn+^x z)2gRGa)U&8x+G#dm!^rjUB;dwc-C5G_l});Ai9EaX$B`aov&k2$}o*YQblw3!+So0 z3gh-k!DH+3kjwR2l51kg0np2@6jDI(n$_;4Qm|v_=K<5^T8rB{=4>k`*hn~=B8MVx zo>GlbgnI@aadU1;knqH@y_|tbLHKF*$q>Bh8#7nBjFcqSi)hTNX#k9%U5s}Aah@0@ zns;}-NH2LVDDO0scfo|NomdF4*8Kd#(NU73jmY;#OGoe?pY4kf7AdiPCrL(sbRjO0 z{3Pc}#>B&ETABddvHRkN50m7j8Z0hpz}Y#T{zd6rlb5vA=K@FEsoklm8=i77$V|8m zX_9BPHEqoiwB(q`7NsIFIw%J09{+_J?svIji*gcmS$=x=*cR-pldA74WkZU#*qJ0e6&KS6X@l{9_Ae{W;r+}EAobcrDYP` zj1Q#Maqh4?*|$F6`-~0R+-jonzhQ9ImB^%~5qCvG=2jzf9dE278SOUhZ6YL06qwDQXD0FK2n2d>_ z9h>18q7btxWQ3BOC87kIsE({4AY6Bs#j?Q8b}bIFqi+Q(qBXE++g4^irdh zq+3JejPr5|VCN}Ea2U*N67f9KPC<)kq!BR0#{cRH`1 zYl*dtb81C#s$%zahNBY253&2*; zsevASc?b>k%LkZjsrJ@=(kHh%9t7a-K_Y9Kvb3{Y>PxC$XPY~m{9B+}!XO%O=4%C} z0p?a~UYR^IIsEdwmufAoStkrV$hwDp_WGFoF?&gC7))s)`V=PR7m9jBu^Py*yi3Ep zM*)i{L|dYuisBytM_Vn0KriE;M(UR~vm-f49LW}J%-<5r?&+vc=^1Ez? zN-$iK>uIdTj3PUZXc+yZtO?b#ai}b0ko)32+@E?}fL$IxMT&-ctGGZG3i;@Up2Mg# z&@lX1q&r>C*oHG%Po#fM$_=7;&7Z zs(}OdEC6?DB=4bhs`f1S7qGL0r1;*v95pjU7cvWZ=$F>M|K;A?xaWb=T;Qm<*J*Spp4Y41kMUaq9}c3`33fum(603y=W_mF;TQ`7I;ViV>7<4R4v>7 zb=li`QP%E3whx|HCtKs`>FF%Eus9;){+{>oJb*EMPk!kK z;7umW_>jrm!26|PobTWGX!3P16QwDIGEa$OI8n{*5Y?$`rkVAd6t@;F{pygdcWbF7 zqzo^_!(V@PG#9yQzxgR6*HfDAFd(b(?X;EX*^V-e08lcC0`(;14=@Swgh3*jk>(Q| z`-Jy_QOqjZ91J#j_54TKP@mdhY!lA?y!R#VL5fkS(XWslHu#cwED>uU%JV(oT4$Bg|QU_eexU(&JB@ie#b#bZ96WJUw) zWqa<2tLUC@5hxXnYl|}+*Dn$geeZePibs{Xu0o~;@=a*(Km1b7 zT1UZXf>B#7GWT?3<40@~?;G_3%wPnjBM23HjaeU}*2%0?^Tt>zy1&lnGTEl z#3{hjXQE6SRHRyf6c(MLO|}zQcL~-^feF?#Iq~9rly+}dM>uY@rl+E_pj24@*MGa= bWB#FUokt+KSayW}{;O4#Rg`_ + # for the operator. + # @notationType -- yaml + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + # -- Priority class for the operator controller. ``system-node-critical`` keeps the + # controller in the same graceful node shutdown phase as MOFED so it can update + # ``network.nvidia.com/operator.mofed.wait`` before the node goes down. See + # https://github.com/Mellanox/network-operator/issues/2954. + priorityClassName: system-node-critical + # -- Termination grace period for the operator controller. Must be greater than + # ``preStopSleepSeconds``. + terminationGracePeriodSeconds: 30 + # -- Seconds to sleep in ``preStop`` before SIGTERM. Keeps the controller reconciling + # while MOFED pods begin terminating during graceful node shutdown. Requires + # Kubernetes >= 1.29 (PodLifecycleSleepAction). Set to ``0`` to disable. + preStopSleepSeconds: 25 + # -- Set additional tolerations for various Daemonsets deployed by the operator. + # @notationType -- yaml + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "" + effect: "NoSchedule" + # -- Configure node selector settings for the operator. + nodeSelector: {} + affinity: + # -- Configure node affinity settings for the operator. + # @notationType -- yaml + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [ "" ] + # -- Network Operator image repository. + repository: nvcr.io/nvidia/cloud-native + # -- Network Operator image name + image: network-operator + # -- Enable Go integration coverage collection (GOCOVERDIR, coverage volume, instrumented image tag). + coverage: + enabled: false + # imagePullSecrets: [] + # -- Name to be used as part of objects name generation. + nameOverride: "" + # -- Name to be used to replace generated names. + fullnameOverride: "" + # tag, if defined will use the given image tag, else Chart. AppVersion will be used. + # tag + # -- Directory, where CNI binaries will be deployed on the nodes. Setting for + # the sriov-network-operator is set with ``sriov-network-operator.cniBinPath`` + # parameter. Note that the CNI bin directory should be aligned with the CNI bin + # directory in the container runtime. + cniBinDirectory: /opt/cni/bin + # -- Directory, where CNI network configuration will be deployed on the nodes. + # Note that the CNI network directory should be aligned with the CNI network + # directory in the container runtime. + cniNetworkDirectory: /etc/cni/net.d + # -- Enable the use of maintenance operator upgrade logic. + maintenanceOperator: + # k8s namespace to be used for generated nodeMaintenance objects + nodeMaintenanceNamespace: default + # enable upgrade controller requestor + useRequestor: false + # upgrade controller requestor ID to be used in nodeMaintenance objects + requestorID: "nvidia.doca-driver-upgrade" + # nodeMaintenance name prefix to be used by upgrade-controller (default is "") + nodeMaintenanceNamePrefix: "network-operator" + # enable drain controller requestor + useDrainControllerRequestor: false + # drain controller requestor ID to be used in nodeMaintenance objects + drainControllerRequestorID: "nvidia.network-operator-drain-controller" + # -- Enable the use of Driver ToolKit to compile DOCA-OFED Drivers (OpenShift only). + useDTK: true + admissionController: + # -- Deploy with admission controller. + enabled: false + # -- Use cert-manager for generating self-signed certificate. + useCertManager: true + # certificate: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + ofedDriver: + initContainer: + # -- Deploy init container. + enable: true + # -- Init container image repository. + repository: nvcr.io/nvidia/mellanox + # -- Init container image name. + image: network-operator-init-container + # -- Init container image version. + version: network-operator-v26.7.0 + + # -- An optional list of references to secrets to use for pulling any of the + # Network Operator images. + imagePullSecrets: [] + + # @ignore + test: + pf: ens2f0 From 9ae409358644bde139242d687eb8a7f8a66619b8 Mon Sep 17 00:00:00 2001 From: Kevin Reeuwijk Date: Thu, 3 Sep 2026 13:27:05 +0200 Subject: [PATCH 2/3] Update docs --- packs/nvidia-network-operator-26.7.0/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packs/nvidia-network-operator-26.7.0/README.md b/packs/nvidia-network-operator-26.7.0/README.md index 69b449c1..19671877 100644 --- a/packs/nvidia-network-operator-26.7.0/README.md +++ b/packs/nvidia-network-operator-26.7.0/README.md @@ -229,6 +229,6 @@ project's github. ## References -- [Official Documentation](https://docs.nvidia.com/networking/software/cloud-orchestration/index.html) +- [Official Documentation](https://docs.nvidia.com/networking/display/kubernetes2670/index.html) - [Operator SDK](https://github.com/operator-framework/operator-sdk) - [GPU-Operator](https://github.com/NVIDIA/gpu-operator) \ No newline at end of file From cba6f0666c2621385b49cb99530733526062d909 Mon Sep 17 00:00:00 2001 From: Kevin Reeuwijk Date: Thu, 3 Sep 2026 13:32:43 +0200 Subject: [PATCH 3/3] Update gitleaks ignore --- .gitleaksignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitleaksignore b/.gitleaksignore index 51c71bee..cad8646f 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -67,3 +67,7 @@ a1b4bf1043a7f3681fdd294579be16aa76a03549:packs/kubewatch-1.0.7/values.yaml:gener 6dcc3f04f80fe8429a878a572bfc59ea94d09f93:packs/sriov-network-operator-1.6.0/values.yaml:private-key:109 c9f6f8d1c46e6bcbde37938b891053075174f1bc:packs/nvidia-nvsentinel-v1.19.0/charts/nvsentinel/charts/mongodb-store/charts/psmdb-db/values.yaml:generic-api-key:675 c9f6f8d1c46e6bcbde37938b891053075174f1bc:packs/nvidia-nvsentinel-v1.19.0/charts/nvsentinel/charts/mongodb-store/charts/psmdb-db/values.yaml:generic-api-key:678 +8f96753560fe5c430a59971bb1161af69ec9bfcb:packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/maintenance-operator-chart/values.yaml:private-key:75 +8f96753560fe5c430a59971bb1161af69ec9bfcb:packs/nvidia-network-operator-26.7.0/charts/network-operator/charts/sriov-network-operator/values.yaml:private-key:94 +8f96753560fe5c430a59971bb1161af69ec9bfcb:packs/nvidia-network-operator-26.7.0/charts/network-operator/values.yaml:private-key:140 +8f96753560fe5c430a59971bb1161af69ec9bfcb:packs/nvidia-network-operator-26.7.0/values.yaml:private-key:179 \ No newline at end of file