From d979a7fb7aa9af51a27fd0bcaeabe205a2549378 Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Mon, 13 Jul 2026 18:30:31 -0400 Subject: [PATCH 1/2] ci(desktop): surface flaky E2E tests instead of retry-masking them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI retries: 2 lets a test fail then pass with no durable signal beyond a one-line console summary — the gap that hid the stream.spec.ts membership race (#1798) for months behind a green checkmark. Adds the json reporter to playwright.config.ts, a script that walks its output for status === "flaky" tests and appends them to the job summary, and changes the smoke/integration artifact-upload steps from if: failure() to if: !cancelled() so retry traces survive a shard that eventually passes. Pass/fail semantics are unchanged. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- .github/workflows/ci.yml | 14 ++++- desktop/.gitignore | 1 + desktop/playwright.config.ts | 1 + desktop/scripts/summarize-flaky-tests.mjs | 71 +++++++++++++++++++++++ 4 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 desktop/scripts/summarize-flaky-tests.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0e85ebfc1..82025d8770 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -234,13 +234,18 @@ jobs: run: just desktop-build - name: Desktop smoke e2e run: cd desktop && pnpm exec playwright test --project=smoke --shard=${{ matrix.shard }}/4 + - name: Summarize flaky tests + if: ${{ !cancelled() }} + run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop Smoke E2E (${{ matrix.shard }})" + working-directory: desktop - name: Upload desktop smoke e2e artifacts - if: failure() + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: desktop-smoke-e2e-artifacts-${{ matrix.shard }} path: | desktop/playwright-report + desktop/playwright-report.json desktop/test-results if-no-files-found: ignore @@ -443,13 +448,18 @@ jobs: run: bash scripts/setup-desktop-test-data.sh - name: Desktop relay-backed e2e run: cd desktop && pnpm exec playwright test --project=integration --shard=${{ matrix.shard }}/2 + - name: Summarize flaky tests + if: ${{ !cancelled() }} + run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop E2E Integration (${{ matrix.shard }}/2)" + working-directory: desktop - name: Upload desktop integration artifacts - if: failure() + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: desktop-e2e-integration-artifacts-${{ matrix.shard }} path: | desktop/playwright-report + desktop/playwright-report.json desktop/test-results /tmp/buzz-relay.log if-no-files-found: ignore diff --git a/desktop/.gitignore b/desktop/.gitignore index 8c39bf90fb..4d3e0c5ac5 100644 --- a/desktop/.gitignore +++ b/desktop/.gitignore @@ -12,6 +12,7 @@ node_modules dist dist-ssr playwright-report +playwright-report.json test-results *.local playwright-report diff --git a/desktop/playwright.config.ts b/desktop/playwright.config.ts index a31306e77d..b118e69e02 100644 --- a/desktop/playwright.config.ts +++ b/desktop/playwright.config.ts @@ -8,6 +8,7 @@ export default defineConfig({ reporter: [ ["list"], ["html", { open: "never", outputFolder: "playwright-report" }], + ["json", { outputFile: "playwright-report.json" }], ], use: { baseURL: "http://127.0.0.1:4173", diff --git a/desktop/scripts/summarize-flaky-tests.mjs b/desktop/scripts/summarize-flaky-tests.mjs new file mode 100644 index 0000000000..4cf17f1460 --- /dev/null +++ b/desktop/scripts/summarize-flaky-tests.mjs @@ -0,0 +1,71 @@ +import { appendFile, readFile } from "node:fs/promises"; + +// Playwright's `retries: 2` (desktop/playwright.config.ts) lets a test fail +// then pass on retry with no durable signal beyond a one-line "N flaky" in +// the console log — the exact gap that hid the stream.spec.ts membership +// race (#1798) for months. This walks the JSON reporter's suite tree +// (recursive: `describe` blocks nest as child `suites`) and appends any +// `status === "flaky"` test to the job's GitHub Actions summary so retried +// failures stay visible even when the shard ultimately goes green. +// +// Usage: node scripts/summarize-flaky-tests.mjs + +function collectFlakyTests(suite, out) { + for (const spec of suite.specs ?? []) { + for (const test of spec.tests ?? []) { + if (test.status !== "flaky") continue; + out.push({ + title: `${suite.file} › ${spec.title}`, + project: test.projectName, + attempts: test.results?.length ?? 0, + }); + } + } + for (const child of suite.suites ?? []) { + collectFlakyTests(child, out); + } +} + +const [reportPath, runLabel] = process.argv.slice(2); +if (!reportPath || !runLabel) { + console.error( + "Usage: node scripts/summarize-flaky-tests.mjs ", + ); + process.exit(1); +} + +// This step runs `if: !cancelled()` purely to surface flaky tests — it must +// never fail the job on its own, so a malformed/unexpected report (from a +// Playwright version bump or a crashed run) is swallowed, not thrown. +try { + const report = JSON.parse(await readFile(reportPath, "utf8")); + + const flaky = []; + for (const suite of report.suites ?? []) { + collectFlakyTests(suite, flaky); + } + + if (flaky.length > 0) { + const escapeCell = (value) => String(value).replaceAll("|", "\\|"); + const rows = flaky + .map( + (t) => + `| ${escapeCell(t.title)} | ${escapeCell(t.project)} | ${t.attempts} |`, + ) + .join("\n"); + const summary = + `### Flaky tests — ${runLabel}\n\n` + + `${flaky.length} test(s) failed at least once before passing on retry:\n\n` + + "| Test | Project | Attempts |\n| --- | --- | --- |\n" + + `${rows}\n`; + + console.log(summary); + + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (summaryFile) { + await appendFile(summaryFile, `${summary}\n`); + } + } +} catch (error) { + console.log(`Skipping flaky-test summary: ${error.message}`); +} From 77b8284937654bc0c8011d70ec675f61bb92b526 Mon Sep 17 00:00:00 2001 From: Will Pfleger Date: Mon, 13 Jul 2026 21:38:12 -0400 Subject: [PATCH 2/2] chore(ci): cap E2E artifact retention to 7 days The smoke and integration Playwright artifact uploads changed from if: failure() to if: !cancelled(), so they now upload on every green run. Cap retention to 7 days so diagnostic traces don't linger at the 90-day default. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82025d8770..6d8b5d378d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,6 +248,7 @@ jobs: desktop/playwright-report.json desktop/test-results if-no-files-found: ignore + retention-days: 7 desktop: name: Desktop @@ -463,6 +464,7 @@ jobs: desktop/test-results /tmp/buzz-relay.log if-no-files-found: ignore + retention-days: 7 - name: Save pnpm store cache if: github.event_name == 'push' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5