Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 61 additions & 7 deletions scripts/weekly-flaky-tests.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ import { mkdir, writeFile } from 'node:fs/promises';
const LOOKBACK_DAYS = 7;
const CONCURRENT_RUNS = 4;
const REPORT_DIR = 'flaky-tests-report';
// This job fails when another job fails. It has no failure of its own.
const GATE_JOB_NAME = 'All required jobs passed or were skipped';
const JOB_TIMEOUT_MESSAGE = 'exceeded the maximum execution time';

function normalizeJobName(name) {
return name
Expand All @@ -24,6 +27,14 @@ function normalizeTestName(name) {
.trim();
}

function newestFirst(runs) {
return [...runs].sort(([a], [b]) => b - a);
}

function runLinks(sortedRuns) {
return sortedRuns.map(([id, url]) => `[${id}](${url})`).join(', ');
}

function markdownCell(value) {
return String(value)
.replace(/&/g, '&')
Expand Down Expand Up @@ -51,6 +62,7 @@ export async function collectReport({ github, context, core, now = new Date() })
}
const runs = listedRuns.filter(run => ['push', 'schedule', 'workflow_dispatch'].includes(run.event));
const tests = new Map();
const steps = new Map();
let failedJobsWithoutTests = 0;
let failedJobs = 0;
let completedRuns = 0;
Expand All @@ -71,11 +83,15 @@ export async function collectReport({ github, context, core, now = new Date() })
filter: 'all',
per_page: 100,
});
const jobsToInspect = jobs.filter(job => job.conclusion === 'failure' && !job.name.includes('(optional)'));
// GitHub reports a job that reached `timeout-minutes` as cancelled.
const jobsToInspect = jobs.filter(
job =>
['failure', 'cancelled'].includes(job.conclusion) &&
!job.name.includes('(optional)') &&
job.name !== GATE_JOB_NAME,
);
core.info(`Run ${run.id}: ${jobs.length} job attempts, ${jobsToInspect.length} failures to inspect.`);
for (const job of jobsToInspect) {
failedJobs++;

let annotations;
try {
annotations = await github.paginate(github.rest.checks.listAnnotations, {
Expand All @@ -91,6 +107,13 @@ export async function collectReport({ github, context, core, now = new Date() })
continue;
}

const timedOut = annotations.some(annotation => annotation.message?.includes(JOB_TIMEOUT_MESSAGE));
// A cancelled job that did not time out was stopped from outside (a person, fail-fast).
if (job.conclusion === 'cancelled' && !timedOut) {
continue;
}
failedJobs++;

const failures = annotations.filter(
annotation =>
annotation.annotation_level === 'failure' &&
Expand All @@ -99,6 +122,20 @@ export async function collectReport({ github, context, core, now = new Date() })
);
if (failures.length === 0) {
failedJobsWithoutTests++;
// A setup step fails in the same way for all E2E applications, so they share one row.
const family = normalizeJobName(job.name).replace(/^E2E .+ Test$/, 'E2E Tests');
const name =
job.steps?.find(step => ['failure', 'cancelled'].includes(step.conclusion))?.name ?? 'No failed step';
const outcome = timedOut ? 'timed out' : 'failed';
const key = JSON.stringify([family, name, outcome]);
let step = steps.get(key);
if (!step) {
step = { family, name, outcome, runs: new Map() };
steps.set(key, step);
}
if (!step.runs.has(run.id)) {
step.runs.set(run.id, job.html_url);
}
}

for (const annotation of failures) {
Expand Down Expand Up @@ -137,6 +174,9 @@ export async function collectReport({ github, context, core, now = new Date() })
a.family.localeCompare(b.family) ||
a.path.localeCompare(b.path),
),
steps: [...steps.values()].sort(
(a, b) => b.runs.size - a.runs.size || a.family.localeCompare(b.family) || a.name.localeCompare(b.name),
),
};
}

Expand All @@ -155,8 +195,7 @@ export function renderReport(report, context) {
} else {
lines.push('| Test | Job | Affected runs | Links to runs | Issue |', '| --- | --- | ---: | --- | --- |');
for (const test of report.tests) {
const runs = [...test.runs].sort(([a], [b]) => b - a);
const links = runs.map(([id, url]) => `[${id}](${url})`).join(', ');
const runs = newestFirst(test.runs);
const issueParams = new URLSearchParams({
template: 'flaky.yml',
title: `[Flaky CI]: ${test.name}`,
Expand All @@ -167,14 +206,29 @@ export function renderReport(report, context) {
});
const issueUrl = `${repositoryUrl}/issues/new?${issueParams}`;
lines.push(
`| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${links} | [Create issue](${issueUrl}) |`,
`| ${markdownCell(test.name)} | ${markdownCell(test.family)} | ${test.runs.size} | ${runLinks(runs)} | [Create issue](${issueUrl}) |`,
);
}
}

// A failure outside a test has no test annotation: a setup step, a crash of the test runner, or a job timeout.
if (report.steps.length > 0) {
lines.push(
'',
'## Weekly job failures outside a test',
'',
'| Job | Step | Result | Affected runs | Links to runs |',
'| --- | --- | --- | ---: | --- |',
...report.steps.map(
step =>
`| ${markdownCell(step.family)} | ${markdownCell(step.name)} | ${step.outcome} | ${step.runs.size} | ${runLinks(newestFirst(step.runs))} |`,
),
);
}

lines.push(
'',
`${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs excluded.`,
`${report.failedJobs} failed job attempts; ${report.failedJobsWithoutTests} without test annotations. Optional jobs and the required-jobs gate excluded.`,
'',
...report.warnings.map(warning => `- ${markdownCell(warning)}`),
);
Expand Down
99 changes: 99 additions & 0 deletions scripts/weekly-flaky-tests.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import { describe, expect, it, vi } from 'vitest';

import { collectReport } from './weekly-flaky-tests.mjs';

const REPOSITORY_URL = 'https://github.com/getsentry/sentry-javascript';
const context = { repo: { owner: 'getsentry', repo: 'sentry-javascript' }, serverUrl: 'https://github.com', runId: 99 };
const now = new Date('2026-11-16T06:00:00Z');

const jobTimeout = {
annotation_level: 'failure',
path: '.github',
message: 'The job has exceeded the maximum execution time of 15m0s',
};
const processExit = { annotation_level: 'failure', path: '.github', message: 'Process completed with exit code 1.' };

function job(id, name, conclusion, steps) {
return {
id,
name,
conclusion,
check_run_url: `https://api.github.com/repos/getsentry/sentry-javascript/check-runs/${id}`,
html_url: `${REPOSITORY_URL}/actions/runs/7/job/${id}`,
steps: steps.map(([stepName, stepConclusion]) => ({ name: stepName, conclusion: stepConclusion })),
};
}

function collect(jobs, annotationsByJob) {
const github = {
rest: {
actions: { listWorkflowRuns: vi.fn(), listJobsForWorkflowRun: vi.fn() },
checks: { listAnnotations: vi.fn() },
},
paginate: vi.fn(async (method, params) => {
if (method === github.rest.actions.listWorkflowRuns) {
return [{ id: 7, event: 'push', conclusion: 'failure', run_attempt: 1 }];
}
if (method === github.rest.actions.listJobsForWorkflowRun) {
return jobs;
}
return annotationsByJob[params.check_run_id] ?? [];
}),
};
return collectReport({ github, context, core: { info: vi.fn() }, now });
}

describe('collectReport', () => {
it('reports the failed step of a job that failed outside a test', async () => {
const report = await collect(
[
job(11, 'Bun Unit Tests', 'failure', [
['Set up Bun', 'failure'],
['Run tests', 'skipped'],
]),
],
{ 11: [processExit] },
);

expect(report.steps).toEqual([
{
family: 'Bun Unit Tests',
name: 'Set up Bun',
outcome: 'failed',
runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/11`]]),
},
]);
expect(report.tests).toEqual([]);
});

it('reports a job that reached its time limit as timed out', async () => {
const report = await collect(
[
job(12, 'Playwright esm chromium Tests', 'cancelled', [
['Restore caches', 'success'],
['Install Playwright', 'cancelled'],
['Run Playwright tests', 'cancelled'],
]),
],
{ 12: [jobTimeout] },
);

expect(report.steps).toEqual([
{
family: 'Playwright Tests',
name: 'Install Playwright',
outcome: 'timed out',
runs: new Map([[7, `${REPOSITORY_URL}/actions/runs/7/job/12`]]),
},
]);
});

it('ignores a cancelled job that did not time out', async () => {
const report = await collect([job(13, 'Lint', 'cancelled', [['Lint source files', 'cancelled']])], {
13: [{ annotation_level: 'failure', path: '.github', message: 'The operation was canceled.' }],
});

expect(report.steps).toEqual([]);
expect(report.failedJobs).toBe(0);
});
});
Loading