From 03bcd4042d2b3b7d089336c62e364225169815dc Mon Sep 17 00:00:00 2001 From: s1gr1d <32902192+s1gr1d@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:52:33 +0100 Subject: [PATCH 1/2] ci: List job failures outside tests in the weekly flaky report The report only counted the failed jobs without a test annotation. In one month, these were about half of the failed develop runs: failed setup steps, and jobs that reached their time limit, which GitHub reports as cancelled. The report now lists them by job and step. The gate job is excluded, because it only repeats the failure of another job. --- scripts/weekly-flaky-tests.mjs | 68 +++++++++-- scripts/weekly-flaky-tests.test.mjs | 169 ++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 scripts/weekly-flaky-tests.test.mjs diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index db00689db3ca..ad0a93c3e001 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -3,6 +3,9 @@ import { mkdir, writeFile } from 'node:fs/promises'; const LOOKBACK_DAYS = 7; const CONCURRENT_RUNS = 4; const REPORT_DIR = 'flaky-tests-report'; +// This job fails when another job fails. It has no failure of its own. +const GATE_JOB_NAME = 'All required jobs passed or were skipped'; +const JOB_TIMEOUT_MESSAGE = 'exceeded the maximum execution time'; function normalizeJobName(name) { return name @@ -24,6 +27,14 @@ function normalizeTestName(name) { .trim(); } +function newestFirst(runs) { + return [...runs].sort(([a], [b]) => b - a); +} + +function runLinks(sortedRuns) { + return sortedRuns.map(([id, url]) => `[${id}](${url})`).join(', '); +} + function markdownCell(value) { return String(value) .replace(/&/g, '&') @@ -51,6 +62,7 @@ export async function collectReport({ github, context, core, now = new Date() }) } const runs = listedRuns.filter(run => ['push', 'schedule', 'workflow_dispatch'].includes(run.event)); const tests = new Map(); + const steps = new Map(); let failedJobsWithoutTests = 0; let failedJobs = 0; let completedRuns = 0; @@ -71,11 +83,15 @@ export async function collectReport({ github, context, core, now = new Date() }) filter: 'all', per_page: 100, }); - const jobsToInspect = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); + // GitHub reports a job that reached `timeout-minutes` as cancelled. + const jobsToInspect = jobs.filter( + job => + ['failure', 'cancelled'].includes(job.conclusion) && + !job.name.includes('(optional)') && + job.name !== GATE_JOB_NAME, + ); core.info(`Run ${run.id}: ${jobs.length} job attempts, ${jobsToInspect.length} failures to inspect.`); for (const job of jobsToInspect) { - failedJobs++; - let annotations; try { annotations = await github.paginate(github.rest.checks.listAnnotations, { @@ -91,6 +107,13 @@ export async function collectReport({ github, context, core, now = new Date() }) continue; } + const timedOut = annotations.some(annotation => annotation.message?.includes(JOB_TIMEOUT_MESSAGE)); + // A cancelled job that did not time out was stopped from outside (a person, fail-fast). + if (job.conclusion === 'cancelled' && !timedOut) { + continue; + } + failedJobs++; + const failures = annotations.filter( annotation => annotation.annotation_level === 'failure' && @@ -99,6 +122,20 @@ export async function collectReport({ github, context, core, now = new Date() }) ); if (failures.length === 0) { failedJobsWithoutTests++; + // A setup step fails in the same way for all E2E applications, so they share one row. + const family = normalizeJobName(job.name).replace(/^E2E .+ Test$/, 'E2E Tests'); + const name = + job.steps?.find(step => ['failure', 'cancelled'].includes(step.conclusion))?.name ?? 'No failed step'; + const outcome = timedOut ? 'timed out' : 'failed'; + const key = JSON.stringify([family, name, outcome]); + let step = steps.get(key); + if (!step) { + step = { family, name, outcome, runs: new Map() }; + steps.set(key, step); + } + if (!step.runs.has(run.id)) { + step.runs.set(run.id, job.html_url); + } } for (const annotation of failures) { @@ -137,6 +174,9 @@ export async function collectReport({ github, context, core, now = new Date() }) a.family.localeCompare(b.family) || a.path.localeCompare(b.path), ), + steps: [...steps.values()].sort( + (a, b) => b.runs.size - a.runs.size || a.family.localeCompare(b.family) || a.name.localeCompare(b.name), + ), }; } @@ -155,8 +195,7 @@ export function renderReport(report, context) { } else { lines.push('| Test | Job | Affected runs | Links to runs | Issue |', '| --- | --- | ---: | --- | --- |'); for (const test of report.tests) { - const runs = [...test.runs].sort(([a], [b]) => b - a); - const links = runs.map(([id, url]) => `[${id}](${url})`).join(', '); + const runs = newestFirst(test.runs); const issueParams = new URLSearchParams({ template: 'flaky.yml', title: `[Flaky CI]: ${test.name}`, @@ -167,14 +206,29 @@ export function renderReport(report, context) { }); const issueUrl = `${repositoryUrl}/issues/new?${issueParams}`; lines.push( - `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} | [Create issue](${issueUrl}) |`, + `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${runLinks(runs)} | [Create issue](${issueUrl}) |`, ); } } + // A failure outside a test has no test annotation: a setup step, a crash of the test runner, or a job timeout. + if (report.steps.length > 0) { + lines.push( + '', + '## Weekly job failures outside a test', + '', + '| Job | Step | Result | Affected runs | Links to runs |', + '| --- | --- | --- | ---: | --- |', + ...report.steps.map( + step => + `| ${markdownCell(step.family)} | ${markdownCell(step.name)} | ${step.outcome} | ${step.runs.size} | ${runLinks(newestFirst(step.runs))} |`, + ), + ); + } + lines.push( '', - `${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs excluded.`, + `${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs and the required-jobs gate excluded.`, '', ...report.warnings.map(warning => `- ${markdownCell(warning)}`), ); diff --git a/scripts/weekly-flaky-tests.test.mjs b/scripts/weekly-flaky-tests.test.mjs new file mode 100644 index 000000000000..0245e212911c --- /dev/null +++ b/scripts/weekly-flaky-tests.test.mjs @@ -0,0 +1,169 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { collectReport, renderReport } from './weekly-flaky-tests.mjs'; + +const REPOSITORY_URL = 'https://github.com/getsentry/sentry-javascript'; +const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 99 }; +const now = new Date('2026-11-16T06:00:00Z'); + +const jobTimeout = { + annotation_level: 'failure', + path: '.github', + message: 'The job has exceeded the maximum execution time of 15m0s', +}; +const processExit = { annotation_level: 'failure', path: '.github', message: 'Process completed with exit code 1.' }; + +function job(id, name, conclusion, steps) { + return { + id, + name, + conclusion, + check_run_url: `https://api.github.com/repos/getsentry/sentry-javascript/check-runs/${id}`, + html_url: `${REPOSITORY_URL}/actions/runs/7/job/${id}`, + steps: steps.map(([stepName, stepConclusion]) => ({ name: stepName, conclusion: stepConclusion })), + }; +} + +function collect(jobs, annotationsByJob) { + const github = { + rest: { + actions: { listWorkflowRuns: vi.fn(), listJobsForWorkflowRun: vi.fn() }, + checks: { listAnnotations: vi.fn() }, + }, + paginate: vi.fn(async (method, params) => { + if (method === github.rest.actions.listWorkflowRuns) { + return [{ id: 7, event: 'push', conclusion: 'failure', run_attempt: 1 }]; + } + if (method === github.rest.actions.listJobsForWorkflowRun) { + return jobs; + } + return annotationsByJob[params.check_run_id] ?? []; + }), + }; + return collectReport({ github, context, core: { info: vi.fn() }, now }); +} + +describe('collectReport', () => { + it('reports the failed step of a job that failed outside a test', async () => { + const report = await collect( + [ + job(11, 'Bun Unit Tests', 'failure', [ + ['Set up Bun', 'failure'], + ['Run tests', 'skipped'], + ]), + ], + { 11: [processExit] }, + ); + + expect(report.steps).toEqual([ + { + family: 'Bun Unit Tests', + name: 'Set up Bun', + outcome: 'failed', + runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/11`]]), + }, + ]); + expect(report.tests).toEqual([]); + }); + + it('reports a job that reached its time limit as timed out', async () => { + const report = await collect( + [ + job(12, 'Playwright esm chromium Tests', 'cancelled', [ + ['Restore caches', 'success'], + ['Install Playwright', 'cancelled'], + ['Run Playwright tests', 'cancelled'], + ]), + ], + { 12: [jobTimeout] }, + ); + + expect(report.steps).toEqual([ + { + family: 'Playwright Tests', + name: 'Install Playwright', + outcome: 'timed out', + runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/12`]]), + }, + ]); + }); + + it('reports a job that reached its time limit after all its steps passed', async () => { + const report = await collect([job(18, 'E2E nuxt-4 Test', 'cancelled', [['Run E2E test', 'success']])], { + 18: [jobTimeout], + }); + + expect(report.steps).toEqual([ + { + family: 'E2E Tests', + name: 'No failed step', + outcome: 'timed out', + runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/18`]]), + }, + ]); + }); + + it('ignores a cancelled job that did not time out', async () => { + const report = await collect([job(13, 'Lint', 'cancelled', [['Lint source files', 'cancelled']])], { + 13: [{ annotation_level: 'failure', path: '.github', message: 'The operation was canceled.' }], + }); + + expect(report.steps).toEqual([]); + expect(report.failedJobs).toBe(0); + }); + + it('ignores the gate job, which only repeats the failure of another job', async () => { + const report = await collect( + [job(14, 'All required jobs passed or were skipped', 'failure', [['Check for failures', 'failure']])], + { 14: [processExit] }, + ); + + expect(report.steps).toEqual([]); + expect(report.failedJobs).toBe(0); + }); + + it('counts the same failed step of two E2E applications as one row', async () => { + const steps = [['Install E2E app dependencies', 'failure']]; + const report = await collect( + [job(15, 'E2E nuxt-4 Test', 'failure', steps), job(16, 'E2E hono-4 (bun) Test', 'failure', steps)], + { 15: [processExit], 16: [processExit] }, + ); + + expect(report.steps).toEqual([ + { + family: 'E2E Tests', + name: 'Install E2E app dependencies', + outcome: 'failed', + runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/15`]]), + }, + ]); + expect(report.failedJobsWithoutTests).toBe(2); + }); +}); + +describe('renderReport', () => { + it('renders a job failure outside a test in its own table', async () => { + const report = await collect( + [job(12, 'Playwright esm chromium Tests', 'cancelled', [['Install Playwright', 'cancelled']])], + { 12: [jobTimeout] }, + ); + + expect(renderReport(report, context).split('\n')).toEqual([ + '## Weekly test failures', + '', + 'Develop · 2026-11-09–2026-11-16 · 1 CI runs.', + '', + 'No test failures found.', + '', + '## Weekly job failures outside a test', + '', + '| Job | Step | Result | Affected runs | Links to runs |', + '| --- | --- | --- | ---: | --- |', + `| Playwright Tests | Install Playwright | timed out | 1 | [7](${REPOSITORY_URL}/actions/runs/7/job/12) |`, + '', + '1 failed job attempts; 1 without test annotations. Optional jobs and the required-jobs gate excluded.', + '', + '', + ]); + }); +}); From 0454eaca14496ac5c8afcb63d6eb4cbcb828b7f0 Mon Sep 17 00:00:00 2001 From: s1gr1d <32902192+s1gr1d@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:30:37 +0100 Subject: [PATCH 2/2] test(ci): Keep only the three tests of the report's job classification The tests that stay cover the logic that is easy to break: a failed job without a test annotation is listed with its failed step, a job that reached its time limit is listed as timed out, and a job that was cancelled for another reason is not listed. The tests for the gate job, the E2E job grouping, the fallback step name, and the rendered table go away. --- scripts/weekly-flaky-tests.test.mjs | 72 +---------------------------- 1 file changed, 1 insertion(+), 71 deletions(-) diff --git a/scripts/weekly-flaky-tests.test.mjs b/scripts/weekly-flaky-tests.test.mjs index 0245e212911c..acce239d1b81 100644 --- a/scripts/weekly-flaky-tests.test.mjs +++ b/scripts/weekly-flaky-tests.test.mjs @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; -import { collectReport, renderReport } from './weekly-flaky-tests.mjs'; +import { collectReport } from './weekly-flaky-tests.mjs'; const REPOSITORY_URL = 'https://github.com/getsentry/sentry-javascript'; const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 99 }; @@ -88,21 +88,6 @@ describe('collectReport', () => { ]); }); - it('reports a job that reached its time limit after all its steps passed', async () => { - const report = await collect([job(18, 'E2E nuxt-4 Test', 'cancelled', [['Run E2E test', 'success']])], { - 18: [jobTimeout], - }); - - expect(report.steps).toEqual([ - { - family: 'E2E Tests', - name: 'No failed step', - outcome: 'timed out', - runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/18`]]), - }, - ]); - }); - it('ignores a cancelled job that did not time out', async () => { const report = await collect([job(13, 'Lint', 'cancelled', [['Lint source files', 'cancelled']])], { 13: [{ annotation_level: 'failure', path: '.github', message: 'The operation was canceled.' }], @@ -111,59 +96,4 @@ describe('collectReport', () => { expect(report.steps).toEqual([]); expect(report.failedJobs).toBe(0); }); - - it('ignores the gate job, which only repeats the failure of another job', async () => { - const report = await collect( - [job(14, 'All required jobs passed or were skipped', 'failure', [['Check for failures', 'failure']])], - { 14: [processExit] }, - ); - - expect(report.steps).toEqual([]); - expect(report.failedJobs).toBe(0); - }); - - it('counts the same failed step of two E2E applications as one row', async () => { - const steps = [['Install E2E app dependencies', 'failure']]; - const report = await collect( - [job(15, 'E2E nuxt-4 Test', 'failure', steps), job(16, 'E2E hono-4 (bun) Test', 'failure', steps)], - { 15: [processExit], 16: [processExit] }, - ); - - expect(report.steps).toEqual([ - { - family: 'E2E Tests', - name: 'Install E2E app dependencies', - outcome: 'failed', - runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/15`]]), - }, - ]); - expect(report.failedJobsWithoutTests).toBe(2); - }); -}); - -describe('renderReport', () => { - it('renders a job failure outside a test in its own table', async () => { - const report = await collect( - [job(12, 'Playwright esm chromium Tests', 'cancelled', [['Install Playwright', 'cancelled']])], - { 12: [jobTimeout] }, - ); - - expect(renderReport(report, context).split('\n')).toEqual([ - '## Weekly test failures', - '', - 'Develop · 2026-11-09–2026-11-16 · 1 CI runs.', - '', - 'No test failures found.', - '', - '## Weekly job failures outside a test', - '', - '| Job | Step | Result | Affected runs | Links to runs |', - '| --- | --- | --- | ---: | --- |', - `| Playwright Tests | Install Playwright | timed out | 1 | [7](${REPOSITORY_URL}/actions/runs/7/job/12) |`, - '', - '1 failed job attempts; 1 without test annotations. Optional jobs and the required-jobs gate excluded.', - '', - '', - ]); - }); });