diff --git a/scripts/weekly-flaky-tests.mjs b/scripts/weekly-flaky-tests.mjs index db00689db3ca..ad0a93c3e001 100644 --- a/scripts/weekly-flaky-tests.mjs +++ b/scripts/weekly-flaky-tests.mjs @@ -3,6 +3,9 @@ import { mkdir, writeFile } from 'node:fs/promises'; const LOOKBACK_DAYS = 7; const CONCURRENT_RUNS = 4; const REPORT_DIR = 'flaky-tests-report'; +// This job fails when another job fails. It has no failure of its own. +const GATE_JOB_NAME = 'All required jobs passed or were skipped'; +const JOB_TIMEOUT_MESSAGE = 'exceeded the maximum execution time'; function normalizeJobName(name) { return name @@ -24,6 +27,14 @@ function normalizeTestName(name) { .trim(); } +function newestFirst(runs) { + return [...runs].sort(([a], [b]) => b - a); +} + +function runLinks(sortedRuns) { + return sortedRuns.map(([id, url]) => `[${id}](${url})`).join(', '); +} + function markdownCell(value) { return String(value) .replace(/&/g, '&') @@ -51,6 +62,7 @@ export async function collectReport({ github, context, core, now = new Date() }) } const runs = listedRuns.filter(run => ['push', 'schedule', 'workflow_dispatch'].includes(run.event)); const tests = new Map(); + const steps = new Map(); let failedJobsWithoutTests = 0; let failedJobs = 0; let completedRuns = 0; @@ -71,11 +83,15 @@ export async function collectReport({ github, context, core, now = new Date() }) filter: 'all', per_page: 100, }); - const jobsToInspect = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)')); + // GitHub reports a job that reached `timeout-minutes` as cancelled. + const jobsToInspect = jobs.filter( + job => + ['failure', 'cancelled'].includes(job.conclusion) && + !job.name.includes('(optional)') && + job.name !== GATE_JOB_NAME, + ); core.info(`Run ${run.id}: ${jobs.length} job attempts, ${jobsToInspect.length} failures to inspect.`); for (const job of jobsToInspect) { - failedJobs++; - let annotations; try { annotations = await github.paginate(github.rest.checks.listAnnotations, { @@ -91,6 +107,13 @@ export async function collectReport({ github, context, core, now = new Date() }) continue; } + const timedOut = annotations.some(annotation => annotation.message?.includes(JOB_TIMEOUT_MESSAGE)); + // A cancelled job that did not time out was stopped from outside (a person, fail-fast). + if (job.conclusion === 'cancelled' && !timedOut) { + continue; + } + failedJobs++; + const failures = annotations.filter( annotation => annotation.annotation_level === 'failure' && @@ -99,6 +122,20 @@ export async function collectReport({ github, context, core, now = new Date() }) ); if (failures.length === 0) { failedJobsWithoutTests++; + // A setup step fails in the same way for all E2E applications, so they share one row. + const family = normalizeJobName(job.name).replace(/^E2E .+ Test$/, 'E2E Tests'); + const name = + job.steps?.find(step => ['failure', 'cancelled'].includes(step.conclusion))?.name ?? 'No failed step'; + const outcome = timedOut ? 'timed out' : 'failed'; + const key = JSON.stringify([family, name, outcome]); + let step = steps.get(key); + if (!step) { + step = { family, name, outcome, runs: new Map() }; + steps.set(key, step); + } + if (!step.runs.has(run.id)) { + step.runs.set(run.id, job.html_url); + } } for (const annotation of failures) { @@ -137,6 +174,9 @@ export async function collectReport({ github, context, core, now = new Date() }) a.family.localeCompare(b.family) || a.path.localeCompare(b.path), ), + steps: [...steps.values()].sort( + (a, b) => b.runs.size - a.runs.size || a.family.localeCompare(b.family) || a.name.localeCompare(b.name), + ), }; } @@ -155,8 +195,7 @@ export function renderReport(report, context) { } else { lines.push('| Test | Job | Affected runs | Links to runs | Issue |', '| --- | --- | ---: | --- | --- |'); for (const test of report.tests) { - const runs = [...test.runs].sort(([a], [b]) => b - a); - const links = runs.map(([id, url]) => `[${id}](${url})`).join(', '); + const runs = newestFirst(test.runs); const issueParams = new URLSearchParams({ template: 'flaky.yml', title: `[Flaky CI]: ${test.name}`, @@ -167,14 +206,29 @@ export function renderReport(report, context) { }); const issueUrl = `${repositoryUrl}/issues/new?${issueParams}`; lines.push( - `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} | [Create issue](${issueUrl}) |`, + `| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${runLinks(runs)} | [Create issue](${issueUrl}) |`, ); } } + // A failure outside a test has no test annotation: a setup step, a crash of the test runner, or a job timeout. + if (report.steps.length > 0) { + lines.push( + '', + '## Weekly job failures outside a test', + '', + '| Job | Step | Result | Affected runs | Links to runs |', + '| --- | --- | --- | ---: | --- |', + ...report.steps.map( + step => + `| ${markdownCell(step.family)} | ${markdownCell(step.name)} | ${step.outcome} | ${step.runs.size} | ${runLinks(newestFirst(step.runs))} |`, + ), + ); + } + lines.push( '', - `${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs excluded.`, + `${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs and the required-jobs gate excluded.`, '', ...report.warnings.map(warning => `- ${markdownCell(warning)}`), ); diff --git a/scripts/weekly-flaky-tests.test.mjs b/scripts/weekly-flaky-tests.test.mjs new file mode 100644 index 000000000000..acce239d1b81 --- /dev/null +++ b/scripts/weekly-flaky-tests.test.mjs @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { collectReport } from './weekly-flaky-tests.mjs'; + +const REPOSITORY_URL = 'https://github.com/getsentry/sentry-javascript'; +const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 99 }; +const now = new Date('2026-11-16T06:00:00Z'); + +const jobTimeout = { + annotation_level: 'failure', + path: '.github', + message: 'The job has exceeded the maximum execution time of 15m0s', +}; +const processExit = { annotation_level: 'failure', path: '.github', message: 'Process completed with exit code 1.' }; + +function job(id, name, conclusion, steps) { + return { + id, + name, + conclusion, + check_run_url: `https://api.github.com/repos/getsentry/sentry-javascript/check-runs/${id}`, + html_url: `${REPOSITORY_URL}/actions/runs/7/job/${id}`, + steps: steps.map(([stepName, stepConclusion]) => ({ name: stepName, conclusion: stepConclusion })), + }; +} + +function collect(jobs, annotationsByJob) { + const github = { + rest: { + actions: { listWorkflowRuns: vi.fn(), listJobsForWorkflowRun: vi.fn() }, + checks: { listAnnotations: vi.fn() }, + }, + paginate: vi.fn(async (method, params) => { + if (method === github.rest.actions.listWorkflowRuns) { + return [{ id: 7, event: 'push', conclusion: 'failure', run_attempt: 1 }]; + } + if (method === github.rest.actions.listJobsForWorkflowRun) { + return jobs; + } + return annotationsByJob[params.check_run_id] ?? []; + }), + }; + return collectReport({ github, context, core: { info: vi.fn() }, now }); +} + +describe('collectReport', () => { + it('reports the failed step of a job that failed outside a test', async () => { + const report = await collect( + [ + job(11, 'Bun Unit Tests', 'failure', [ + ['Set up Bun', 'failure'], + ['Run tests', 'skipped'], + ]), + ], + { 11: [processExit] }, + ); + + expect(report.steps).toEqual([ + { + family: 'Bun Unit Tests', + name: 'Set up Bun', + outcome: 'failed', + runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/11`]]), + }, + ]); + expect(report.tests).toEqual([]); + }); + + it('reports a job that reached its time limit as timed out', async () => { + const report = await collect( + [ + job(12, 'Playwright esm chromium Tests', 'cancelled', [ + ['Restore caches', 'success'], + ['Install Playwright', 'cancelled'], + ['Run Playwright tests', 'cancelled'], + ]), + ], + { 12: [jobTimeout] }, + ); + + expect(report.steps).toEqual([ + { + family: 'Playwright Tests', + name: 'Install Playwright', + outcome: 'timed out', + runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/12`]]), + }, + ]); + }); + + it('ignores a cancelled job that did not time out', async () => { + const report = await collect([job(13, 'Lint', 'cancelled', [['Lint source files', 'cancelled']])], { + 13: [{ annotation_level: 'failure', path: '.github', message: 'The operation was canceled.' }], + }); + + expect(report.steps).toEqual([]); + expect(report.failedJobs).toBe(0); + }); +});