From e043cc6c6c775576b067edabdc0e0405affc63b9 Mon Sep 17 00:00:00 2001
From: qe-ci-ai-bot <qe-ci-ai-bot@redhat.com>
Date: Wed, 3 Jun 2026 13:41:28 +0000
Subject: [PATCH] fix(ci): count JUnit errors in Slack failure report
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Slack alert was reporting "0 tests failed" even when tests actually
failed. The root cause is that the JUnit XML spec distinguishes
"failures" (assertion failures) from "errors" (exceptions/timeouts).
Playwright reports TimeoutError and similar issues as `errors`, not
`failures`. The old code only parsed the `failures` attribute, so any
test that failed due to a timeout or exception was silently uncounted.

Fix by summing both `failures` and `errors` from the root <testsuites>
element. Also add a safety net: if Playwright exits non-zero but the
JUnit XML somehow reports 0 failures and 0 errors, report "some" instead
of the misleading "0".

Verified against build 2061711989599637504 where showcase had
failures="0" errors="4" and showcase-runtime had failures="0" errors="8"
— both incorrectly reported as "0 tests failed" on Slack.

Requested-By: <@U05QPKB8VB3> (gliraesi)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .ci/pipelines/lib/testing.sh | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.ci/pipelines/lib/testing.sh b/.ci/pipelines/lib/testing.sh
index d23519036d..55faffe792 100644
--- a/.ci/pipelines/lib/testing.sh
+++ b/.ci/pipelines/lib/testing.sh
@@ -161,8 +161,22 @@ testing::run_tests() {
   if [[ "${test_result}" -eq 0 ]]; then
     failed_tests="0"
   elif [[ -f "${e2e_tests_dir}/${JUNIT_RESULTS}" ]]; then
-    failed_tests=$(grep -oP 'failures="\K[0-9]+' "${e2e_tests_dir}/${JUNIT_RESULTS}" | head -n 1)
-    failed_tests="${failed_tests:-some}"
+    # JUnit XML distinguishes "failures" (assertion failures) from "errors"
+    # (exceptions/timeouts). Playwright reports TimeoutError, crash, and
+    # similar issues as errors, not failures. Sum both from the root
+    # <testsuites> element so the Slack notification reflects the real count.
+    local _junit_failures _junit_errors
+    _junit_failures=$(grep -oP 'failures="\K[0-9]+' "${e2e_tests_dir}/${JUNIT_RESULTS}" | head -n 1)
+    _junit_errors=$(grep -oP 'errors="\K[0-9]+' "${e2e_tests_dir}/${JUNIT_RESULTS}" | head -n 1)
+    _junit_failures="${_junit_failures:-0}"
+    _junit_errors="${_junit_errors:-0}"
+    failed_tests=$((_junit_failures + _junit_errors))
+    if [[ "${failed_tests}" -eq 0 ]]; then
+      # Playwright exited non-zero but JUnit reports 0 failures and 0 errors —
+      # the process likely crashed or timed out globally. Report "some" so the
+      # Slack alert doesn't misleadingly say "0 tests failed".
+      failed_tests="some"
+    fi
     echo "Number of failed tests: ${failed_tests}"
   else
     echo "JUnit results file not found: ${e2e_tests_dir}/${JUNIT_RESULTS}"