From 329e89a815ac6e78ec0e7fd651e1ae1230d2dffe Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 20 Jan 2026 17:29:54 +0100 Subject: [PATCH] test/extended/prometheus: move alert tests to "Test Framework" This commit renames 2 tests related to alerting rules: * 1 test checking that no rule fails to evaluate. * 1 test checking that there's no firing alert other than approved exceptions. When these tests fail, the associated Jira used to be assigned to the Monitoring component because of the "sig-instrumentation" prefix. But in practice, it can be any component and monitoring is (often) only the messenger. The tests are renamed with the [Jira:"Test Framework"] prefix instead which redirects the tickets to TRT. Signed-off-by: Simon Pasquier --- test/extended/prometheus/prometheus.go | 124 ++++++++++++++----------- 1 file changed, 69 insertions(+), 55 deletions(-) diff --git a/test/extended/prometheus/prometheus.go b/test/extended/prometheus/prometheus.go index c64e4af1397c..579a135302bb 100644 --- a/test/extended/prometheus/prometheus.go +++ b/test/extended/prometheus/prometheus.go @@ -619,6 +619,75 @@ var _ = g.Describe("[sig-instrumentation][Late] Alerts", func() { }) }) +var _ = g.Describe("[Jira:\"Test Framework\"] Prometheus [apigroup:config.openshift.io]", func() { + defer g.GinkgoRecover() + var ( + oc = exutil.NewCLIWithPodSecurityLevel("prometheus", admissionapi.LevelBaseline) + ) + + g.BeforeEach(func(ctx g.SpecContext) { + err := exutil.WaitForAnImageStream( + oc.AdminImageClient().ImageV1().ImageStreams("openshift"), "tools", + exutil.CheckImageStreamLatestTagPopulated, exutil.CheckImageStreamTagNotFound) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.Describe("when installed on the cluster", func() { + g.It("shouldn't have failing rules evaluation", func(ctx g.SpecContext) { + // we only consider samples since the beginning of the test + testDuration := exutil.DurationSinceStartInSeconds().String() + + tests := map[string]bool{ + fmt.Sprintf(`increase(prometheus_rule_evaluation_failures_total[%s]) >= 1`, testDuration): false, + } + err := helper.RunQueries(ctx, oc.NewPrometheusClient(ctx), tests, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("should have no unexpected alerts in firing state [Early][apigroup:config.openshift.io]", func(ctx g.SpecContext) { + allowedAlertNames := sets.New[string](allowedalerts.AllowedAlertNames...) + + // Checking Watchdog alert state is done in "should have a Watchdog alert in firing state". + // we exclude alerts that have their own separate tests. + for _, alertTest := range allowedalerts.AllAlertTests(&platformidentification.JobType{}, nil, allowedalerts.DefaultAllowances) { + allowedAlertNames.Insert(alertTest.AlertName()) + } + + if exutil.IsNoUpgradeFeatureSet(oc) { + // On a TechPreviewNoUpgrade or CustomNoUpgrade cluster we must ignore the TechPreviewNoUpgrade and ClusterNotUpgradeable alerts generated by the CVO. + // These two alerts are expected in this case when a cluster is configured to enable Tech Preview features, + // as they were intended to be "gentle reminders" to the cluster admins of the ramifications of enabling Tech Preview + allowedAlertNames.Insert("TechPreviewNoUpgrade", "ClusterNotUpgradeable") + } + + // OSD-26887: managed services taints several nodes as infrastructure. This taint appears to be applied + // after some of the platform DS are scheduled there, causing this alert to fire. Managed services + // rebalances the DS after the taint is added, and the alert clears, but origin fails this test. Allowing + // this alert to fire while we investigate why the taint is not added at node birth. + isManagedService, err := exutil.IsManagedServiceCluster(ctx, oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isManagedService { + allowedAlertNames.Insert("KubeDaemonSetMisScheduled") + } + // https://issues.redhat.com/browse/OCPBUGS-48340 + if SkipOperatorHubMetricsCheck(oc) { + allowedAlertNames.Insert("OperatorHubSourceError") + } + + tests := map[string]bool{ + // openshift-e2e-loki alerts should never fail this test, we've seen this happen on daemon set rollout stuck when CI loki was down. + // + // CCXDEV-16087 / OCPBUGS-77314: Insights call API that fails with 504 due to load + // The unless clause excludes KubeJobFailed alerts from periodic-gathering jobs in openshift-insights + // TODO: Revert (or remove the unless clause) once CCXDEV-16087 / OCPBUGS-77314 is addressed + fmt.Sprintf(`(ALERTS{alertname!~"%s",alertstate="firing",severity!="info",namespace!="openshift-e2e-loki"} unless ALERTS{alertname="KubeJobFailed",alertstate="firing",namespace="openshift-insights",job_name=~"periodic-gathering-.*"}) >= 1`, strings.Join(sets.List(allowedAlertNames), "|")): false, + } + err = helper.RunQueries(ctx, oc.NewPrometheusClient(ctx), tests, oc) + o.Expect(err).NotTo(o.HaveOccurred()) + }) + }) +}) + var _ = g.Describe("[sig-instrumentation] Prometheus [apigroup:image.openshift.io]", func() { defer g.GinkgoRecover() ctx := context.TODO() @@ -868,61 +937,6 @@ var _ = g.Describe("[sig-instrumentation] Prometheus [apigroup:image.openshift.i o.Expect(err).NotTo(o.HaveOccurred()) }) - g.It("shouldn't have failing rules evaluation", func() { - // we only consider samples since the beginning of the test - testDuration := exutil.DurationSinceStartInSeconds().String() - - tests := map[string]bool{ - fmt.Sprintf(`increase(prometheus_rule_evaluation_failures_total[%s]) >= 1`, testDuration): false, - } - err := helper.RunQueries(context.TODO(), oc.NewPrometheusClient(context.TODO()), tests, oc) - o.Expect(err).NotTo(o.HaveOccurred()) - }) - - g.It("shouldn't report any alerts in firing state apart from Watchdog and AlertmanagerReceiversNotConfigured [Early][apigroup:config.openshift.io]", func() { - // Copy so we can expand: - allowedAlertNames := make([]string, len(allowedalerts.AllowedAlertNames)) - copy(allowedAlertNames, allowedalerts.AllowedAlertNames) - - // Checking Watchdog alert state is done in "should have a Watchdog alert in firing state". - // we exclude alerts that have their own separate tests. - for _, alertTest := range allowedalerts.AllAlertTests(&platformidentification.JobType{}, nil, allowedalerts.DefaultAllowances) { - allowedAlertNames = append(allowedAlertNames, alertTest.AlertName()) - } - - if exutil.IsNoUpgradeFeatureSet(oc) { - // On a TechPreviewNoUpgrade or CustomNoUpgrade cluster we must ignore the TechPreviewNoUpgrade and ClusterNotUpgradeable alerts generated by the CVO. - // These two alerts are expected in this case when a cluster is configured to enable Tech Preview features, - // as they were intended to be "gentle reminders" to the cluster admins of the ramifications of enabling Tech Preview - allowedAlertNames = append(allowedAlertNames, "TechPreviewNoUpgrade", "ClusterNotUpgradeable") - } - - // OSD-26887: managed services taints several nodes as infrastructure. This taint appears to be applied - // after some of the platform DS are scheduled there, causing this alert to fire. Managed services - // rebalances the DS after the taint is added, and the alert clears, but origin fails this test. Allowing - // this alert to fire while we investigate why the taint is not added at node birth. - isManagedService, err := exutil.IsManagedServiceCluster(ctx, oc.AdminKubeClient()) - o.Expect(err).NotTo(o.HaveOccurred()) - if isManagedService { - allowedAlertNames = append(allowedAlertNames, "KubeDaemonSetMisScheduled") - } - // https://issues.redhat.com/browse/OCPBUGS-48340 - if SkipOperatorHubMetricsCheck(oc) { - allowedAlertNames = append(allowedAlertNames, "OperatorHubSourceError") - } - - tests := map[string]bool{ - // openshift-e2e-loki alerts should never fail this test, we've seen this happen on daemon set rollout stuck when CI loki was down. - // - // CCXDEV-16087 / OCPBUGS-77314: Insights call API that fails with 504 due to load - // The unless clause excludes KubeJobFailed alerts from periodic-gathering jobs in openshift-insights - // TODO: Revert (or remove the unless clause) once CCXDEV-16087 / OCPBUGS-77314 is addressed - fmt.Sprintf(`(ALERTS{alertname!~"%s",alertstate="firing",severity!="info",namespace!="openshift-e2e-loki"} unless ALERTS{alertname="KubeJobFailed",alertstate="firing",namespace="openshift-insights",job_name=~"periodic-gathering-.*"}) >= 1`, strings.Join(allowedAlertNames, "|")): false, - } - err = helper.RunQueries(context.TODO(), oc.NewPrometheusClient(context.TODO()), tests, oc) - o.Expect(err).NotTo(o.HaveOccurred()) - }) - g.It("should provide ingress metrics", func() { var lastErrs []error o.Expect(wait.PollUntilContextTimeout(context.Background(), 10*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) {