From f5f5b07ca47a60a555357c8f2bc3ac3b37ad7c86 Mon Sep 17 00:00:00 2001 From: Kieran Osgood Date: Tue, 18 Aug 2026 11:18:48 +0100 Subject: [PATCH] fix(e2e): improve GitHub report diagnostics --- e2e/lib/e2e_github_reporter.rb | 21 +++++++++--- e2e/test/e2e_github_reporter_test.rb | 49 ++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/e2e/lib/e2e_github_reporter.rb b/e2e/lib/e2e_github_reporter.rb index 3b34da15e..9bb349bea 100644 --- a/e2e/lib/e2e_github_reporter.rb +++ b/e2e/lib/e2e_github_reporter.rb @@ -91,14 +91,19 @@ def check_run_payload def results_table lines = [] - lines << "| Status | Suite | Target | Platform | OS version tag | Device |" + lines << "| Status | Tags | Target | Platform | OS version tag | Device |" lines << "|---|---|---|---|---|---|" @results.each do |result| - lines << "| #{status_icon(result)} | `#{result["application_id"]}` | #{result["target"]} | #{result["platform"]} | #{result["os_version_tag"]} | #{device_cell(result)} |" + lines << "| #{status_icon(result)} | `#{tags_cell(result)}` | #{result["target"]} | #{result["platform"]} | #{result["os_version_tag"]} | #{device_cell(result)} |" end lines end + def tags_cell(result) + tags = result.fetch("include_tags", []).to_a + tags.empty? ? "all" : tags.join(", ") + end + def completeness_lines blocked? ? blocked_lines : shortfall_lines end @@ -225,7 +230,7 @@ def missing_runs end def missing_run_label(run) - "`#{run["application_id"] || run["target"]}` (#{run["platform"]})" + "`#{run["application_id"] || run["target"]}` · #{tags_cell(run)} (#{run["platform"]})" end def missing_count @@ -263,6 +268,7 @@ def failure_details(result) lines << "" lines << "### #{failure_heading(result)}" lines << "" + lines.concat(setup_error_lines(result)) lines << "| Test | Status | Artifacts |" lines << "|---|---|---|" tests = result.fetch("failed_tests", []) @@ -276,8 +282,15 @@ def failure_details(result) lines end + def setup_error_lines(result) + return [] if blank?(result["error"]) + + message = result["error"].to_s.split("\n").map(&:strip).reject(&:empty?).join(" ") + ["> `#{result["error_class"] || "Error"}`: #{message}", ""] + end + def failure_heading(result) - "#{os_label(result["platform"])} — #{result["application_id"]}" + "#{os_label(result["platform"])} — #{result["target"]}" end def artifact_links(testcase, result) diff --git a/e2e/test/e2e_github_reporter_test.rb b/e2e/test/e2e_github_reporter_test.rb index a69c7c042..6a3f5958a 100644 --- a/e2e/test/e2e_github_reporter_test.rb +++ b/e2e/test/e2e_github_reporter_test.rb @@ -35,7 +35,8 @@ def swift_ios_run "target" => "swift", "platform" => "ios", "os_version_tag" => "latest", - "execute" => "." + "execute" => ".", + "include_tags" => ["launch"] } end @@ -46,7 +47,8 @@ def react_native_ios_run "target" => "react-native", "platform" => "ios", "os_version_tag" => "latest", - "execute" => "." + "execute" => ".", + "include_tags" => ["launch"] } end @@ -120,14 +122,14 @@ def test_blocked_report_names_the_failed_stage_and_lists_the_skipped_runs assert_includes summary, "> - `e2e-build-react-native-ios` — [build log]" assert_includes summary, "/build/a7111bcd)" assert_includes summary, "> None of the 2 planned runs executed:" - assert_includes summary, "> - `swift-ios` (ios)" + assert_includes summary, "> - `swift-ios` · launch (ios)" assert_includes summary, "> [Pipeline build](https://app.bitrise.io/app/" end def test_blocked_report_omits_the_empty_tables body = blocked_reporter.comment_body - refute_includes body, "| Status | Suite |" + refute_includes body, "| Status | Tags |" refute_includes body, "## Install this build" refute_includes body, "| SDK | Install |" end @@ -140,16 +142,26 @@ def test_missing_runs_are_named_without_a_stage_roster summary = reporter(results: [], run_plan: [swift_ios_run], expected: 1).markdown_summary assert_includes summary, "did not report" - assert_includes summary, "> - `swift-ios` (ios)" + assert_includes summary, "> - `swift-ios` · launch (ios)" refute_includes summary, "[!CAUTION]" end - def test_results_table_suite_column_shows_the_application_id_not_the_execute_path - reported = swift_ios_run.merge("passed" => true, "resolved_device" => "iPhone") + def test_results_table_names_the_selected_tags + reported = swift_ios_run.merge( + "passed" => true, + "resolved_device" => "iPhone", + "include_tags" => %w[cart checkout] + ) body = reporter(results: [reported]).comment_body - assert_includes body, "| ✅ | `swift-ios` |" - refute_includes body, "| ✅ | `.` |" + assert_includes body, "| Status | Tags | Target | Platform | OS version tag | Device |" + assert_includes body, "| ✅ | `cart, checkout` |" + end + + def test_results_table_reads_all_without_selected_tags + reported = swift_ios_run.merge("passed" => true, "resolved_device" => "iPhone", "include_tags" => []) + + assert_includes reporter(results: [reported]).comment_body, "| ✅ | `all` |" end def test_partial_report_keeps_the_table_and_names_the_failed_stage @@ -161,7 +173,7 @@ def test_partial_report_keeps_the_table_and_names_the_failed_stage expected: 2 ).markdown_summary - assert_includes summary, "| Status | Suite |" + assert_includes summary, "| Status | Tags |" assert_includes summary, "did not report" assert_includes summary, "> 1 pipeline stage failed:" assert_includes summary, "> - `e2e-execute-browserstack-run` — [build log]" @@ -204,4 +216,21 @@ def test_complete_run_has_no_missing_run_lines refute_includes summary, "did not report" end + + def test_failure_heading_names_the_target + failed = swift_ios_run.merge("passed" => false, "failed_tests" => []) + + assert_includes reporter(results: [failed]).markdown_summary, "### iOS — swift" + end + + def test_setup_error_names_the_class_and_message + failed = swift_ios_run.merge( + "passed" => false, + "failed_tests" => [], + "error_class" => "RuntimeError", + "error" => "first line\nsecond line" + ) + + assert_includes reporter(results: [failed]).markdown_summary, "> `RuntimeError`: first line second line" + end end