From 3ca880dba69dbee63b2c7fe6c01bd94124ed80f2 Mon Sep 17 00:00:00 2001 From: Nishant Chawla Date: Mon, 10 Aug 2026 13:25:54 +0530 Subject: [PATCH 1/3] SPLAT-2830: Add CI changes and periodic PROW changes for CI-Cluster Capacity monitoring --- ...am-ci-cluster-monitor-main__periodics.yaml | 35 +++++++ ...eam-ci-cluster-monitor-main-periodics.yaml | 91 +++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml b/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml index cf285f93d2f68..e62d05f686380 100644 --- a/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml +++ b/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml @@ -73,6 +73,41 @@ tests: requests: cpu: 100m memory: 256Mi +- as: capacity-monitor + cluster: build01 + cron: 0 */6 * * * + reporter_config: + channel: '#forum-ocp-splat-alerts' + job_states_to_report: + - failure + - error + report_template: Nutanix cluster capacity threshold exceeded. <{{.Status.URL}}|View + logs> + steps: + cluster_profile: nutanix + test: + - as: monitor + commands: | + set +x + source /tmp/nutanix-creds/secrets.sh + mkdir -p /tmp/nutanix-auth + echo -n "${prism_central_username}" > /tmp/nutanix-auth/nutanix-username + echo -n "${prism_central_password}" > /tmp/nutanix-auth/nutanix-password + set -x + make build + ./bin/capacity-monitor --config config/nutanix.yaml \ + --credentials-dir /tmp/nutanix-auth \ + --endpoint "${prism_central_host}" \ + --port "${prism_central_port}" + credentials: + - mount_path: /tmp/nutanix-creds + name: nutanix-pc + namespace: test-credentials + from: src + resources: + requests: + cpu: 100m + memory: 256Mi zz_generated_metadata: branch: main org: openshift-splat-team diff --git a/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml b/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml index 031a9eb81a323..d76ff6c3d867c 100644 --- a/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml +++ b/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml @@ -1,4 +1,95 @@ periodics: +- agent: kubernetes + cluster: build01 + cron: 0 */6 * * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: main + org: openshift-splat-team + repo: ci-cluster-monitor + labels: + ci-operator.openshift.io/cloud: nutanix + ci-operator.openshift.io/cloud-cluster-profile: nutanix + ci-operator.openshift.io/cluster: build01 + ci-operator.openshift.io/variant: periodics + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-splat-team-ci-cluster-monitor-main-periodics-capacity-monitor + reporter_config: + slack: + channel: '#forum-ocp-splat-alerts' + job_states_to_report: + - failure + - error + report_template: Nutanix cluster capacity threshold exceeded. <{{.Status.URL}}|View + logs> + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=capacity-monitor + - --variant=periodics + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build12 cron: 0 9 * * 1-5 From dce02c1bc447e957600d5a90f588ecf490b17413 Mon Sep 17 00:00:00 2001 From: Nishant Chawla Date: Mon, 10 Aug 2026 13:53:24 +0530 Subject: [PATCH 2/3] fixes --- ...openshift-splat-team-ci-cluster-monitor-main__periodics.yaml | 2 +- .../openshift-splat-team-ci-cluster-monitor-main-periodics.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml b/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml index e62d05f686380..8411a78f0d068 100644 --- a/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml +++ b/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml @@ -81,7 +81,7 @@ tests: job_states_to_report: - failure - error - report_template: Nutanix cluster capacity threshold exceeded. <{{.Status.URL}}|View + report_template: Capacity Monitor alert - check logs for details. <{{.Status.URL}}|View logs> steps: cluster_profile: nutanix diff --git a/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml b/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml index d76ff6c3d867c..e073b46b1097b 100644 --- a/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml +++ b/ci-operator/jobs/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main-periodics.yaml @@ -23,7 +23,7 @@ periodics: job_states_to_report: - failure - error - report_template: Nutanix cluster capacity threshold exceeded. <{{.Status.URL}}|View + report_template: Capacity Monitor alert - check logs for details. <{{.Status.URL}}|View logs> spec: containers: From 251b94828e700a8d097722811a32a3e2c1f0bd83 Mon Sep 17 00:00:00 2001 From: Nishant Chawla Date: Mon, 10 Aug 2026 15:50:39 +0530 Subject: [PATCH 3/3] fixes --- ...ift-splat-team-ci-cluster-monitor-main__periodics.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml b/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml index 8411a78f0d068..3ad91af6af02d 100644 --- a/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml +++ b/ci-operator/config/openshift-splat-team/ci-cluster-monitor/openshift-splat-team-ci-cluster-monitor-main__periodics.yaml @@ -52,9 +52,12 @@ tests: test: - as: monitor commands: | + set -euo pipefail set +x source /tmp/nutanix-creds/secrets.sh + umask 077 mkdir -p /tmp/nutanix-auth + trap 'rm -rf /tmp/nutanix-auth' EXIT echo -n "${prism_central_username}" > /tmp/nutanix-auth/nutanix-username echo -n "${prism_central_password}" > /tmp/nutanix-auth/nutanix-password set -x @@ -69,6 +72,7 @@ tests: name: nutanix-pc namespace: test-credentials from: src + grace_period: 30s resources: requests: cpu: 100m @@ -88,9 +92,12 @@ tests: test: - as: monitor commands: | + set -euo pipefail set +x source /tmp/nutanix-creds/secrets.sh + umask 077 mkdir -p /tmp/nutanix-auth + trap 'rm -rf /tmp/nutanix-auth' EXIT echo -n "${prism_central_username}" > /tmp/nutanix-auth/nutanix-username echo -n "${prism_central_password}" > /tmp/nutanix-auth/nutanix-password set -x @@ -104,6 +111,7 @@ tests: name: nutanix-pc namespace: test-credentials from: src + grace_period: 30s resources: requests: cpu: 100m