From e68eef5aca456aff33596e30a9856b8d0801c298 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 02:12:30 +0000 Subject: [PATCH 1/4] fork CI: one automatic re-run for known infrastructure failures only (#130) A failed Fork tests run gets one GitHub "re-run failed jobs", only when every failed job shows an infrastructure sign: lost runner, full memory box (OOM kills, memory.max hits, memory.high throttling while stalled), or a time limit hit while the machine was stalled. Never for a test failure, never twice, fail closed on unreadable signs. - fork-ci-pressure.py: start/end steps in each app job of fork-tests.yml; prints the memory pressure numbers in the log and leaves them as a notice when the job did not succeed. Never fails the job. - fork-ci-rerun.py + fork-ci-rerun.json: the decision with every rule in one script and every project fact in config; warning annotation and job summary per re-run, with a 24 h count. - fork-ci-rerun.yml: scheduled sweep (offset from the fleet sweep) plus workflow_dispatch for one run. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-rerun.json | 35 ++ .github/fork-ci-rerun.md | 111 +++++ .github/fork-ci-tests.txt | 2 + .github/fork-owned-paths.txt | 1 + .github/scripts/fork-ci-pressure.py | 204 +++++++++ .github/scripts/fork-ci-pressure.test.py | 220 +++++++++ .github/scripts/fork-ci-rerun.py | 419 +++++++++++++++++ .github/scripts/fork-ci-rerun.test.py | 559 +++++++++++++++++++++++ .github/workflows/fork-ci-rerun.yml | 61 +++ .github/workflows/fork-tests.yml | 84 ++++ 10 files changed, 1696 insertions(+) create mode 100644 .github/fork-ci-rerun.json create mode 100644 .github/fork-ci-rerun.md create mode 100644 .github/scripts/fork-ci-pressure.py create mode 100644 .github/scripts/fork-ci-pressure.test.py create mode 100644 .github/scripts/fork-ci-rerun.py create mode 100644 .github/scripts/fork-ci-rerun.test.py create mode 100644 .github/workflows/fork-ci-rerun.yml diff --git a/.github/fork-ci-rerun.json b/.github/fork-ci-rerun.json new file mode 100644 index 0000000000000..9707660272815 --- /dev/null +++ b/.github/fork-ci-rerun.json @@ -0,0 +1,35 @@ +{ + "_doc": [ + "Fork only: when a failed Fork tests run gets its one automatic re-run. Read by .github/scripts/fork-ci-rerun.py", + "(.github/workflows/fork-ci-rerun.yml every 10 minutes); the script has no fact about this repository, these are", + "all of them. Runbook: .github/fork-ci-rerun.md.", + "signs: regular expressions matched against a failed job's annotations (GitHub's own messages).", + " lost: the runner went away; enough on its own.", + " limit: the job or a step ran past its time limit; only with the job's pressure note showing `stalled`.", + "stalled: the job's memory pressure note (.github/scripts/fork-ci-pressure.py) shows the machine or the job's box", + " stalled: some-or-all-tasks-waiting (full) share of the job's time, or the 5-minute full average at its end.", + "memory_full: any failure, when the note shows the box full: OOM kills, memory.max hits, or memory.high", + " throttling with a full stall share at least high_events_with_full_share_percent.", + "skip_when: with this environment variable (the workflow sets it from a repository variable) at one of these", + " values, a run that qualifies is only reported, not re-run (fleet mode: a re-run of failed jobs would wait for", + " fleet runners that refuse it, .github/fork-fleet.md)." + ], + "repository": "scuttletoby/t3code", + "workflow": "fork-tests.yml", + "actors": ["scuttletoby"], + "events": ["push", "pull_request"], + "push_branches": ["main"], + "followers": ["fork-ci-result"], + "window_hours": 24, + "catch_up_minutes": 60, + "skip_when": { "variable": "CI_RUNNERS", "is": ["fleet"] }, + "signs": { + "lost": ["lost communication with the server", "The runner has received a shutdown signal"], + "limit": [ + "has exceeded the maximum execution time of", + "The action '[^']*' has timed out after" + ] + }, + "stalled": { "full_share_percent": 10, "full_avg300_percent": 10 }, + "memory_full": { "oom_kills": 1, "max_events": 1, "high_events_with_full_share_percent": 10 } +} diff --git a/.github/fork-ci-rerun.md b/.github/fork-ci-rerun.md new file mode 100644 index 0000000000000..01658ac4db025 --- /dev/null +++ b/.github/fork-ci-rerun.md @@ -0,0 +1,111 @@ +# Fork CI re-run: one automatic re-run for infrastructure failures only + +A failed `Fork tests` run gets **one** automatic "re-run failed jobs" (GitHub's +own), only when every failed job shows a known infrastructure sign. Never for a +test failure, never twice: a repeat red is left red, and it goes to an issue +and its fix first. + +| Part | What it does | +| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.github/scripts/fork-ci-pressure.py` | Two steps in each app job of `fork-tests.yml` (the routable jobs of `fork-fleet.json`): `start` after the checkout, `end` last (`if: always()`). Facts only, never fails the job. | +| `.github/scripts/fork-ci-rerun.py` | The decision, with every rule in one place, and the re-run. | +| `.github/fork-ci-rerun.json` | Every fact and number the decision uses: repository, workflow, trusted actors and branches, followers, sign patterns, thresholds, the fleet-mode switch. | +| `.github/workflows/fork-ci-rerun.yml` | Runs the decision every 10 minutes (minutes 5, 15, ...), GitHub-hosted, `actions: write`, `checks: read`. `workflow_dispatch` with `run_id` checks one run. | +| `fork-ci-rerun.test.py`, `fork-ci-pressure.test.py` | Each sign, each refusal, the state cases below, the wiring. | + +## The numbers in the job log + +The `end` step prints one line in every app job, passed or not: + +``` +fork-ci pressure: job failure; 612 s since start; memory stall full 14.2% some 30.1% of the job; +full avg10/60/300 22.0/18.4/12.9%; available 310 MB, swap used 1024 MB, OOM kills 1; +cgroup /fleet.slice/fork-fleet-x.service: high None MB max None MB use 2900 MB, events high None max None oom_kill None, full 15.0%; ... +``` + +- **stall full / some**: the share of the job's time in which all (full) or + some tasks of the machine waited for memory (`/proc/pressure/memory`, from + its `total` counters between `start` and `end`). +- **avg10/60/300**: the kernel's own full-stall averages at the end. +- **OOM kills**: `/proc/vmstat` `oom_kill` since `start`. +- **cgroup**: the job's own cgroup and each ancestor with a memory limit (the + box: a slice or unit with `MemoryHigh` or `MemoryMax`), with limit, use, + `memory.events` (high, max, oom_kill) since `start` and its own stall share. + +A job that did not succeed also leaves the same numbers as one notice +annotation (`fork-ci pressure`), which the decision reads from GitHub after the +run. `null` (or `None` in the line) is a number that could not be read: no +start sample, a reboot in between, a file the runner cannot see. + +## The rules (fork-ci-rerun.py) + +1. Only a trusted run: this repository, `fork-tests.yml`, started and + triggered by a listed actor, a push to a listed branch or a pull request + from a branch of this repository. +2. Never twice: if any attempt of the run was triggered by someone else (the + re-run is triggered by `github-actions`), the run is left red. +3. Each failed job (failure, cancelled, timed out) except the followers + (`fork-ci-result`, which needs every job) must show a sign: + - **lost**: GitHub says the runner lost communication with the server, or + received a shutdown signal. Enough on its own (no note needed: the job + never reached its last step). + - **memory full**: the note shows an OOM kill on the machine or in the box, + the box hitting `memory.max`, or `memory.high` throttling while stalled + (full) at least `high_events_with_full_share_percent` of the job. + - **limit while stalled**: GitHub says the job or a step ran past its time + limit, and the note shows the machine or the box stalled (full) at least + `full_share_percent` of the job, or a 5-minute full average at least + `full_avg300_percent` at the end. +4. One failed job without a sign leaves the run red: a test failure, a cancel + by a person or a newer push, a job with no note or a garbled one, more than + one note. Fail closed. +5. The branch must still be at the run's commit (a newer push tests it anyway). +6. With `CI_RUNNERS=fleet` (`skip_when`), a run that qualifies is reported with + an error annotation and not re-run: a re-run of failed jobs keeps its fleet + labels and would wait for fleet runners that refuse the bot's attempt + (`fork-fleet.md`). Re-run it by hand. + +## Visible and counted + +- Each re-run is a **warning annotation** on the `Fork CI re-run` run, with + the job names, the signs, GitHub's message and the numbers. +- Each pass writes a **job summary**: a table of every red run of the last hour + and what was done (re-run, left red and why), and the count of runs whose + latest attempt was an automatic re-run in the last 24 hours (this one's and + the fleet sweep's). A stalled machine keeps showing up there and in the + failed job's own log line; the re-run does not hide it. + +## State + +Nothing is kept between passes: each one reads GitHub. + +| Case | What happens | +| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| A pass runs again on a run it re-ran | The run is queued or running (skipped), or ended again with an attempt by `github-actions`: left red. | +| Two passes (or this and the fleet sweep) race | The run is read again just before the request. If GitHub refuses the second request and the run has a new attempt, that is fine. | +| A pass stops before its request reached GitHub | The run is unchanged; the next pass (within `catch_up_minutes`) sends it once. | +| A pass stops after GitHub took the request | The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | +| An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | +| A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). | +| GitHub errors on one run | A warning; the other runs are still handled, and the pass ends red so the error is seen. The next pass tries again. | + +## Overlap with the fleet sweep + +`fork-fleet.yml`'s sweep also re-runs (the whole run, GitHub-hosted) a run +whose fleet runner was lost, in `auto` mode. Both stay at one re-run per run: +each leaves alone a run with an attempt by `github-actions`, and GitHub refuses +a re-run while one is going. They are offset by 5 minutes. + +## Limits + +- No live proof before merge: scheduled workflows run from the default branch. + After merge, dispatch `Fork CI re-run` once and read its summary. +- An OOM kill by a user-space killer (earlyoom, systemd-oomd) is not in + `memory.events` or `oom_kill`: it shows only as a stall, if at all. +- GitHub keeps at most 10 notices per step and 50 annotations per job: on a + job with very many annotations the note can be dropped. The job then reads + as "no note" and stays red (fail closed). +- A trusted run's own code could print a forged note. Only trusted runs are + read, and two notes in one job are refused. +- A job a step's own time limit failed counts as `limit` only with a stall; a + slow test on a calm machine stays red. diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 2303a0eabbf70..845549929c882 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -33,6 +33,8 @@ # is a note; one that was present and then deleted fails. .github/scripts/fork-ci-tests.test.cjs ~11s .github/scripts/fork-checks-ran.test.py +.github/scripts/fork-ci-pressure.test.py +.github/scripts/fork-ci-rerun.test.py .github/scripts/fork-contracts.test.py ~45s .github/scripts/fork-fleet-desk-agent.test.py ~23s .github/scripts/fork-fleet-desk-key.test.py ~10s diff --git a/.github/fork-owned-paths.txt b/.github/fork-owned-paths.txt index 0a70eacb284d8..0802773458c93 100644 --- a/.github/fork-owned-paths.txt +++ b/.github/fork-owned-paths.txt @@ -24,6 +24,7 @@ fork-kit/ .github/fork-* # Read by app jobs or the runner routing every app job uses. !.github/scripts/fork-fleet.py +!.github/scripts/fork-ci-pressure.py !.github/scripts/fork-test-shards.cjs !.github/scripts/fork-vitest-gc-check.cjs !.github/fork-fleet.json diff --git a/.github/scripts/fork-ci-pressure.py b/.github/scripts/fork-ci-pressure.py new file mode 100644 index 0000000000000..c1c4861af3954 --- /dev/null +++ b/.github/scripts/fork-ci-pressure.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Fork only: a job's memory pressure numbers, for the log and for .github/scripts/fork-ci-rerun.py. + + python3 -I fork-ci-pressure.py start right after a job's checkout: saves a sample to $RUNNER_TEMP. + python3 -I fork-ci-pressure.py end the job's last step (if: always(), JOB_STATUS=${{ job.status }}): prints + one log line with the numbers since `start`, and, when the job did not + succeed, the same numbers as one notice annotation (NOTE_TITLE, NOTE_HEAD + + compact JSON) that fork-ci-rerun.py reads from GitHub after the run. + +Facts only, no decision: what counts as a stalled or full machine is fork-ci-rerun.py's, from its config. The numbers: +- the machine's memory pressure (/proc/pressure/memory, PSI): the share of the job's time that some or all tasks + waited for memory (from the `total` counters), and the 10 s, 60 s and 300 s averages at the end; +- /proc/meminfo's available memory and swap in use, and /proc/vmstat's OOM kill count since `start`; +- the job's own cgroup (v2) and each ancestor with a memory limit (the box the job runs in: a systemd slice or + unit with MemoryHigh or MemoryMax): limit, use, and memory.events' high, max and oom_kill since `start`, and its + own pressure share. +Never fails the job: a file it cannot read is null, a missing start sample makes every "since start" number null, +and any other error prints a warning and exits 0. Standard library only. +""" +import json, os, sys, time + +NOTE_TITLE = 'fork-ci pressure' +NOTE_HEAD = 'fork-ci-pressure v1 ' +SAMPLE = 'fork-ci-pressure-start.json' +EVENTS = ('high', 'max', 'oom', 'oom_kill') + + +def read(path): + try: + with open(path, encoding='utf-8') as f: + return f.read() + except (OSError, UnicodeDecodeError): + return None + + +def psi(text): + """/proc/pressure/memory or a cgroup's memory.pressure: {"some": {...}, "full": {...}} (avg in %, total in µs).""" + if text is None: + return None + out = {} + for line in text.splitlines(): + kind, _, rest = line.partition(' ') + if kind not in ('some', 'full'): + continue + fields = {} + for part in rest.split(): + key, _, value = part.partition('=') + try: + fields[key] = float(value) if key.startswith('avg') else int(value) + except ValueError: + pass + out[kind] = fields + return out or None + + +def keyed(text, scale=1): + """'Key: 123 kB' or 'key 123' lines as {key: number}.""" + if text is None: + return None + out = {} + for line in text.splitlines(): + parts = line.replace(':', ' ').split() + if len(parts) >= 2: + try: + out[parts[0]] = int(parts[1]) // scale + except ValueError: + pass + return out + + +def limit(text): + """memory.max / memory.high: bytes, or None for 'max' (no limit) or unreadable.""" + text = (text or '').strip() + return int(text) if text.isdigit() else None + + +def own_cgroup(proc): + """The cgroup v2 path of this process ('0::/a/b' in /proc/self/cgroup), or None.""" + for line in (read(os.path.join(proc, 'self/cgroup')) or '').splitlines(): + if line.startswith('0::'): + return line[3:].strip() or '/' + return None + + +def cgroups(proc, root): + """The own cgroup and every ancestor with a memory limit, innermost first.""" + path = own_cgroup(proc) + out = [] + while path: + where = os.path.join(root, path.lstrip('/')) + high, top = limit(read(os.path.join(where, 'memory.high'))), limit(read(os.path.join(where, 'memory.max'))) + if not out or high is not None or top is not None: + events = keyed(read(os.path.join(where, 'memory.events'))) + current = read(os.path.join(where, 'memory.current')) + out.append({'path': path, 'high_bytes': high, 'max_bytes': top, + 'current_bytes': int(current) if (current or '').strip().isdigit() else None, + 'events': {k: events[k] for k in EVENTS if k in events} if events else None, + 'psi': psi(read(os.path.join(where, 'memory.pressure')))}) + if path == '/': + break + path = os.path.dirname(path.rstrip('/')) or '/' + return out + + +def sample(proc='/proc', root='/sys/fs/cgroup', now=time.time): + vm = keyed(read(os.path.join(proc, 'vmstat'))) or {} + return {'t': now(), 'boot': (read(os.path.join(proc, 'sys/kernel/random/boot_id')) or '').strip() or None, + 'psi': psi(read(os.path.join(proc, 'pressure/memory'))), + 'mem_kb': keyed(read(os.path.join(proc, 'meminfo'))), 'oom_kill': vm.get('oom_kill'), + 'cgroups': cgroups(proc, root)} + + +def share(a, b, kind, seconds): + """% of `seconds` that `kind` (some/full) pressure counted between two PSI readings, or None.""" + try: + return round((b[kind]['total'] - a[kind]['total']) / 1e6 / seconds * 100, 1) + except (KeyError, TypeError, ZeroDivisionError): + return None + + +def delta(a, b): + return b - a if isinstance(a, int) and isinstance(b, int) and b >= a else None + + +def mb(n): + return None if n is None else n // (1024 * 1024) + + +def numbers(start, end): + """The note: what fork-ci-rerun.py reads. Every 'since start' number is null without a usable start sample.""" + t = start.get('t') if isinstance(start, dict) else None + same = isinstance(t, (int, float)) and start.get('boot') == end.get('boot') and end['t'] > t + start = start if same else {} + seconds = end['t'] - t if same else None + psi_end, mem = end.get('psi') or {}, end.get('mem_kb') or {} + full, some = psi_end.get('full') or {}, psi_end.get('some') or {} + swap = (mem['SwapTotal'] - mem['SwapFree']) // 1024 if 'SwapTotal' in mem and 'SwapFree' in mem else None + host = {'full_share': share(start.get('psi'), psi_end, 'full', seconds) if same else None, + 'some_share': share(start.get('psi'), psi_end, 'some', seconds) if same else None, + 'full_avg10': full.get('avg10'), 'full_avg60': full.get('avg60'), 'full_avg300': full.get('avg300'), + 'some_avg60': some.get('avg60'), 'some_avg300': some.get('avg300'), + 'available_mb': mem['MemAvailable'] // 1024 if 'MemAvailable' in mem else None, 'swap_used_mb': swap, + 'oom_kill': delta(start.get('oom_kill'), end.get('oom_kill')) if same else None} + before = {c['path']: c for c in (start.get('cgroups') or [])} if same else {} + boxes = [] + for c in end.get('cgroups') or []: + b = before.get(c['path']) or {} + ev_a, ev_b = b.get('events') or {}, c.get('events') or {} + boxes.append({'path': c['path'], 'high_mb': mb(c.get('high_bytes')), 'max_mb': mb(c.get('max_bytes')), + 'current_mb': mb(c.get('current_bytes')), + **{k: delta(ev_a.get(k), ev_b.get(k)) for k in EVENTS}, + 'full_share': share(b.get('psi'), c.get('psi'), 'full', seconds) if b else None}) + return {'seconds': None if seconds is None else int(seconds), 'host': host, 'cgroups': boxes} + + +def line(status, n): + h = n['host'] + out = [f'fork-ci pressure: job {status}', f'{n["seconds"]} s since start' if n['seconds'] is not None + else 'no start sample (numbers since start unknown)', + f'memory stall full {h["full_share"]}% some {h["some_share"]}% of the job', + f'full avg10/60/300 {h["full_avg10"]}/{h["full_avg60"]}/{h["full_avg300"]}%', + f'available {h["available_mb"]} MB, swap used {h["swap_used_mb"]} MB, OOM kills {h["oom_kill"]}'] + for c in n['cgroups']: + out.append(f'cgroup {c["path"]}: high {c["high_mb"]} MB max {c["max_mb"]} MB use {c["current_mb"]} MB, ' + f'events high {c["high"]} max {c["max"]} oom_kill {c["oom_kill"]}, full {c["full_share"]}%') + return '; '.join(out) + + +def command(text): + """A workflow command's data, escaped as the runner wants (%, CR, LF).""" + return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A') + + +def main(argv, env=os.environ, out=sys.stdout, proc='/proc', root='/sys/fs/cgroup', now=time.time): + verb = argv[1] if len(argv) == 2 else '' + temp = env.get('RUNNER_TEMP') or '' + if verb not in ('start', 'end') or not temp: + print('::warning::fork-ci pressure: usage: fork-ci-pressure.py start|end, with RUNNER_TEMP set', file=out) + return 0 + path = os.path.join(temp, SAMPLE) + try: + now_sample = sample(proc, root, now) + if verb == 'start': + with open(path, 'w', encoding='utf-8') as f: + json.dump(now_sample, f) + print(f'fork-ci pressure: start sample saved ({len(now_sample["cgroups"])} cgroup(s))', file=out) + return 0 + try: + start = json.loads(read(path) or 'null') + except ValueError: + start = None + n = numbers(start if isinstance(start, dict) else {}, now_sample) + status = (env.get('JOB_STATUS') or 'unknown').strip() + print(line(status, n), file=out) + if status != 'success': + note = json.dumps({'status': status, **n}, separators=(',', ':'), sort_keys=True) + print(f'::notice title={NOTE_TITLE}::{command(NOTE_HEAD + note)}', file=out) + except Exception as e: # never fail the job + print(f'::warning::fork-ci pressure: {verb}: {e.__class__.__name__}: {e}', file=out) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/.github/scripts/fork-ci-pressure.test.py b/.github/scripts/fork-ci-pressure.test.py new file mode 100644 index 0000000000000..996872ca3da94 --- /dev/null +++ b/.github/scripts/fork-ci-pressure.test.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Tests of fork-ci-pressure.py (a job's memory pressure numbers), with a fake /proc and cgroup tree, and of its +steps in fork-tests.yml. Run: python3 .""" +import io, json, os, re, shutil, subprocess, sys, tempfile, unittest +import importlib.util + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SCRIPT = os.path.join(HERE, 'fork-ci-pressure.py') +_spec = importlib.util.spec_from_file_location('fork_ci_pressure', SCRIPT) +probe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(probe) + +# The jobs that run app builds and tests on a fleet or GitHub-hosted runner (each routable job of fork-fleet.json). +with open(os.path.join(ROOT, '.github/fork-fleet.json'), encoding='utf-8') as f: + APP_JOBS = sorted({j['job'] for j in json.load(f)['jobs']}) +GIB = 1024 ** 3 + + +def psi_text(some_total, full_total, full_avg10=0.0, full_avg300=0.0): + return (f'some avg10=1.00 avg60=0.50 avg300=0.20 total={some_total}\n' + f'full avg10={full_avg10:.2f} avg60=0.00 avg300={full_avg300:.2f} total={full_total}\n') + + +class Machine: + """A fake /proc and /sys/fs/cgroup: the job in /fleet.slice/job.service, the slice limited to 8 GiB.""" + + def __init__(self, d): + self.d, self.proc, self.root = d, os.path.join(d, 'proc'), os.path.join(d, 'cgroup') + self.write('proc/self/cgroup', '0::/fleet.slice/job.service\n') + self.write('proc/sys/kernel/random/boot_id', 'boot-1\n') + self.write('proc/meminfo', 'MemTotal: 16000000 kB\nMemAvailable: 8192000 kB\n' + 'SwapTotal: 2097152 kB\nSwapFree: 1048576 kB\n') + self.write('proc/vmstat', 'nr_free_pages 1\noom_kill 3\n') + self.write('proc/pressure/memory', psi_text(0, 0)) + self.box('fleet.slice', high='max', top=str(8 * GIB), events=(5, 0, 0, 0), psi=psi_text(0, 0)) + self.box('fleet.slice/job.service', high='max', top='max', events=(0, 0, 0, 0), psi=psi_text(0, 0)) + self.write('cgroup/fleet.slice/job.service/memory.current', str(3 * GIB)) + + def write(self, rel, text): + path = os.path.join(self.d, rel) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + f.write(text) + + def box(self, rel, high=None, top=None, events=None, psi=None): + base = f'cgroup/{rel}' + if high is not None: + self.write(f'{base}/memory.high', high + '\n') + if top is not None: + self.write(f'{base}/memory.max', top + '\n') + if events is not None: + self.write(f'{base}/memory.events', ''.join(f'{k} {v}\n' for k, v in zip(probe.EVENTS, events))) + if psi is not None: + self.write(f'{base}/memory.pressure', psi) + + +def run(m, verb, temp, status=None, t=1000.0): + out = io.StringIO() + env = {'RUNNER_TEMP': temp} + if status: + env['JOB_STATUS'] = status + code = probe.main(['p', verb], env=env, out=out, proc=m.proc, root=m.root, now=lambda: t) + return code, out.getvalue() + + +def the_note(out): + lines = [l for l in out.splitlines() if l.startswith('::notice ')] + if not lines: + return None + assert len(lines) == 1, lines + data = lines[0].split('::', 2)[2].replace('%0A', '\n').replace('%0D', '\r').replace('%25', '%') + assert data.startswith(probe.NOTE_HEAD), data + return json.loads(data[len(probe.NOTE_HEAD):]) + + +class Probe(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.d) + self.m = Machine(self.d) + self.temp = os.path.join(self.d, 'temp') + os.makedirs(self.temp) + + def stall(self): + """600 s later: the machine stalled 60 s (10%), the slice OOM-killed twice and hit memory.max.""" + self.m.write('proc/pressure/memory', psi_text(120_000_000, 60_000_000, full_avg10=30.0, full_avg300=12.5)) + self.m.write('proc/vmstat', 'nr_free_pages 1\noom_kill 5\n') + self.m.box('fleet.slice', events=(9, 4, 2, 2), psi=psi_text(0, 90_000_000)) + + def test_a_failed_job_prints_its_numbers_and_one_note(self): + self.assertEqual(run(self.m, 'start', self.temp)[0], 0) + self.stall() + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(code, 0) + self.assertIn('fork-ci pressure: job failure; 600 s since start; memory stall full 10.0% some 20.0%', out) + self.assertIn('full avg10/60/300 30.0/0.0/12.5%', out) + self.assertIn('available 8000 MB, swap used 1024 MB, OOM kills 2', out) + self.assertIn('cgroup /fleet.slice: high None MB max 8192 MB', out) + n = the_note(out) + self.assertEqual((n['status'], n['seconds']), ('failure', 600)) + self.assertEqual((n['host']['full_share'], n['host']['full_avg300'], n['host']['oom_kill']), (10.0, 12.5, 2)) + own, slice_ = n['cgroups'] + self.assertEqual(own['path'], '/fleet.slice/job.service') + self.assertEqual(own['current_mb'], 3072) + self.assertEqual((slice_['path'], slice_['max_mb'], slice_['high'], slice_['max'], slice_['oom_kill'], + slice_['full_share']), ('/fleet.slice', 8192, 4, 4, 2, 15.0)) + self.assertNotIn('/', [c['path'] for c in n['cgroups']], 'the root has no limit: not listed') + + def test_a_passing_job_prints_its_numbers_but_no_note(self): + run(self.m, 'start', self.temp) + code, out = run(self.m, 'end', self.temp, 'success', t=1600.0) + self.assertEqual(code, 0) + self.assertIn('fork-ci pressure: job success', out) + self.assertIsNone(the_note(out)) + + def test_a_cancelled_job_and_an_unknown_status_leave_a_note(self): + run(self.m, 'start', self.temp) + for status in ('cancelled', None): + n = the_note(run(self.m, 'end', self.temp, status, t=1600.0)[1]) + self.assertEqual(n['status'], status or 'unknown') + + def test_no_start_sample_makes_every_since_start_number_null(self): + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) # the start step never ran + self.assertEqual(code, 0) + self.assertIn('no start sample', out) + n = the_note(out) + self.assertIsNone(n['seconds']) + self.assertEqual([n['host'][k] for k in ('full_share', 'some_share', 'oom_kill')], [None, None, None]) + self.assertTrue(all(c['oom_kill'] is None and c['full_share'] is None for c in n['cgroups'])) + self.assertEqual(n['cgroups'][1]['max_mb'], 8192, 'what is read at the end is still there') + + def test_a_garbled_or_foreign_start_sample_counts_as_none(self): + for text in ('{"t": ', '[1]', '{"t": "soon"}', json.dumps({'t': 500.0, 'boot': 'boot-0'})): + with self.subTest(text): + with open(os.path.join(self.temp, probe.SAMPLE), 'w') as f: + f.write(text) + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(code, 0) + self.assertIsNone(the_note(out)['host']['full_share'], out) + + def test_a_reboot_between_start_and_end_counts_nothing_since_start(self): + run(self.m, 'start', self.temp) + self.stall() + self.m.write('proc/sys/kernel/random/boot_id', 'boot-2\n') + n = the_note(run(self.m, 'end', self.temp, 'failure', t=1600.0)[1]) + self.assertIsNone(n['host']['oom_kill']) + + def test_counters_that_went_back_are_null_not_negative(self): + run(self.m, 'start', self.temp) + self.m.write('proc/vmstat', 'oom_kill 1\n') + n = the_note(run(self.m, 'end', self.temp, 'failure', t=1600.0)[1]) + self.assertIsNone(n['host']['oom_kill']) + + def test_nothing_readable_still_ends_well(self): + shutil.rmtree(self.m.root) + for rel in ('pressure', 'meminfo', 'vmstat', 'self'): + path = os.path.join(self.m.proc, rel) + (shutil.rmtree if os.path.isdir(path) else os.unlink)(path) + self.assertEqual(run(self.m, 'start', self.temp)[0], 0) + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(code, 0) + n = the_note(out) + self.assertEqual(n['cgroups'], []) + self.assertIsNone(n['host']['full_avg10']) + + def test_a_cgroup_path_with_odd_characters_stays_one_annotation(self): + self.m.write('proc/self/cgroup', '0::/odd%25\\x2dslice\n') + self.m.box('odd%25\\x2dslice', high=str(GIB), events=(1, 0, 0, 0)) + run(self.m, 'start', self.temp) + code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) + self.assertEqual(the_note(out)['cgroups'][0]['path'], '/odd%25\\x2dslice') + + def test_it_never_fails_the_job(self): + out = io.StringIO() + self.assertEqual(probe.main(['p', 'start'], env={}, out=out), 0) + self.assertIn('::warning::', out.getvalue()) + self.assertEqual(probe.main(['p', 'nonsense'], env={'RUNNER_TEMP': self.temp}, out=io.StringIO()), 0) + os.chmod(self.temp, 0o500) # the sample cannot be written + self.addCleanup(os.chmod, self.temp, 0o700) + if os.geteuid() != 0: + code, text = run(self.m, 'start', self.temp) + self.assertEqual(code, 0) + self.assertIn('::warning::fork-ci pressure: start', text) + r = subprocess.run([sys.executable, '-I', SCRIPT, 'end'], capture_output=True, text=True, + env={'PATH': os.environ.get('PATH', ''), 'RUNNER_TEMP': self.temp, 'JOB_STATUS': 'failure'}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('fork-ci pressure: job failure', r.stdout) + + +class Steps(unittest.TestCase): + """Each app job of fork-tests.yml takes a sample right after its checkout and prints the numbers last.""" + + def test_every_app_job_has_the_start_and_end_steps(self): + import yaml + with open(os.path.join(ROOT, '.github/workflows/fork-tests.yml'), encoding='utf-8') as f: + wf = yaml.safe_load(f) + self.assertTrue(APP_JOBS) + for name in APP_JOBS: + with self.subTest(name): + steps = wf['jobs'][name]['steps'] + self.assertEqual(steps[0]['name'], 'Checkout') + self.assertEqual(steps[1]['run'], 'python3 -I .github/scripts/fork-ci-pressure.py start') + self.assertNotIn('if', steps[1]) + end = steps[-1] + self.assertEqual(end['run'], 'python3 -I .github/scripts/fork-ci-pressure.py end') + self.assertTrue(re.fullmatch(r"\$\{\{ always\(\) && hashFiles\('\.github/scripts/fork-ci-pressure" + r"\.py'\) != '' \}\}", end['if']), end['if']) + self.assertEqual(end['env'], {'JOB_STATUS': '${{ job.status }}'}) + self.assertEqual(sum('fork-ci-pressure' in str(s.get('run')) for s in steps), 2) + + def test_the_probe_is_not_fork_owned_app_jobs_read_it(self): + with open(os.path.join(ROOT, '.github/fork-owned-paths.txt'), encoding='utf-8') as f: + self.assertIn('!.github/scripts/fork-ci-pressure.py', f.read().splitlines()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/fork-ci-rerun.py b/.github/scripts/fork-ci-rerun.py new file mode 100644 index 0000000000000..4454941e5c96c --- /dev/null +++ b/.github/scripts/fork-ci-rerun.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""Fork only: one automatic re-run of a failed Fork tests run, only for a known infrastructure sign. + + python3 -I fork-ci-rerun.py sweep [--run-id N] .github/workflows/fork-ci-rerun.yml (every 10 minutes, GitHub-hosted, + actions: write, checks: read): each failed trusted run of the last + catch_up_minutes (or only run N) whose every failed job shows a sign + gets GitHub's own "re-run failed jobs", once. + python3 -I fork-ci-rerun.py check checks CONFIG (fork-ci-rerun.test.py runs it). + +THE rules, in one place (the facts, patterns and numbers are CONFIG's, .github/fork-ci-rerun.json; none is here): +- Only a trusted run: of CONFIG's repository and workflow, started and triggered by one of its actors, a push to one + of its branches or a pull request whose head is this repository. Anything else is left alone. +- Never twice: a run any of whose attempts was triggered by someone else (GitHub's re-run from this script, or the + fleet sweep's, is triggered by github-actions) is left red. A repeat red goes to an issue and its fix first. +- Each failed job (failure, cancelled, timed out) but CONFIG's followers (jobs that need all the others and fail + only because one of them did) must show a sign, from its annotations (GitHub's own messages) and its pressure + note (fork-ci-pressure.py): + lost the runner lost communication or was shut down (CONFIG signs.lost): enough on its own; + memory full any failure, when the note shows the job's box or machine full (CONFIG memory_full); + limit a job or step past its time limit (CONFIG signs.limit), only when the note shows the machine or + box stalled (CONFIG stalled). + One failed job without a sign (a test failure, a person's cancel, no note, a garbled note) leaves the run red: + fail closed. A run whose branch has moved on, or is gone, is not re-run. +- With CONFIG skip_when (fleet mode), a run that qualifies is only reported, with an error annotation. +Visible and counted: each re-run is a warning annotation with its signs and numbers, a line in this job's summary, +and every pass counts the runs a bot re-ran in the window. Nothing is kept between passes: each reads GitHub, so a +pass that stopped half-way is finished by the next, and one that runs again finds the bot's attempt and does nothing. +Runbook: .github/fork-ci-rerun.md. Standard library only. +""" +import argparse, calendar, http.client, json, os, re, sys, time +import urllib.error, urllib.parse, urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +CONFIG = os.path.join(os.path.dirname(HERE), 'fork-ci-rerun.json') +API = os.environ.get('GITHUB_API_URL') or 'https://api.github.com' +NOTE_TITLE = 'fork-ci pressure' # fork-ci-pressure.py's +NOTE_HEAD = 'fork-ci-pressure v1 ' +RED = ('failure', 'cancelled', 'timed_out') +CHECK_RUN = re.compile(r'/check-runs/([0-9]{1,20})$') + + +def say(msg, out=None): + print(msg, file=out or sys.stdout, flush=True) + + +# ---------------------------------------------------------------- config + +def load(path=CONFIG): + with open(path, encoding='utf-8') as f: + cfg = json.load(f) + problems = check_config(cfg) + if problems: + raise ValueError(f'{path}: ' + '; '.join(problems)) + return cfg + + +def number(x): + return isinstance(x, (int, float)) and not isinstance(x, bool) + + +def names(x): + return isinstance(x, list) and bool(x) and all(isinstance(s, str) and s for s in x) + + +def check_config(cfg): + out = [] + if not isinstance(cfg, dict): + return ['not a JSON object'] + known = {'_doc', 'repository', 'workflow', 'actors', 'events', 'push_branches', 'followers', 'window_hours', + 'catch_up_minutes', 'skip_when', 'signs', 'stalled', 'memory_full'} + out += [f'unknown key {k!r}' for k in sorted(set(cfg) - known)] + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', str(cfg.get('repository', ''))): + out.append('repository must be owner/name') + if not re.fullmatch(r'[A-Za-z0-9_.-]+\.ya?ml', str(cfg.get('workflow', ''))): + out.append('workflow must be a workflow file name') + for key in ('actors', 'events', 'push_branches'): + if not names(cfg.get(key)): + out.append(f'{key} must be a list of names') + if not set(cfg.get('events') or []) <= {'push', 'pull_request'}: + out.append('events: only push and pull_request') + if not isinstance(cfg.get('followers'), list) or not all(isinstance(s, str) for s in cfg.get('followers', [])): + out.append('followers must be a list of job names') + for key in ('window_hours', 'catch_up_minutes'): + if not number(cfg.get(key)) or cfg[key] <= 0: + out.append(f'{key} must be a positive number') + skip = cfg.get('skip_when') + if not (isinstance(skip, dict) and re.fullmatch(r'[A-Z][A-Z0-9_]*', str(skip.get('variable', ''))) + and names(skip.get('is'))): + out.append('skip_when must be {"variable": NAME, "is": [values]}') + signs = cfg.get('signs') + if not isinstance(signs, dict) or set(signs) != {'lost', 'limit'}: + out.append('signs must have exactly lost and limit') + else: + for kind, patterns in signs.items(): + if not names(patterns): + out.append(f'signs.{kind} must be a list of patterns') + continue + for p in patterns: + try: + re.compile(p) + except re.error as e: + out.append(f'signs.{kind}: {p!r}: {e}') + for key, fields in (('stalled', ('full_share_percent', 'full_avg300_percent')), + ('memory_full', ('oom_kills', 'max_events', 'high_events_with_full_share_percent'))): + got = cfg.get(key) + if not isinstance(got, dict) or set(got) != set(fields) or not all(number(got[f]) and got[f] > 0 + for f in fields): + out.append(f'{key} must have exactly {", ".join(fields)}, each a positive number') + return out + + +# ---------------------------------------------------------------- GitHub + +class GitHub: + def __init__(self, token, repo): + self.token, self.repo = token, repo + + def call(self, path, method='GET', data=None): + """path: below /repos//.""" + if not self.token: + raise ValueError(f'no token for {path}') + body = None if data is None else json.dumps(data).encode() + req = urllib.request.Request(f'{API}/repos/{self.repo}/{path}', data=body, method=method, + headers={'Accept': 'application/vnd.github+json', 'User-Agent': 'fork-ci-rerun', + 'Authorization': f'Bearer {self.token}'}) + with urllib.request.urlopen(req, timeout=30) as r: + got = r.read() + return json.loads(got) if got else {} + + def pages(self, path, key, most=10): + out = [] + for page in range(1, most + 1): + got = self.call(f'{path}{"&" if "?" in path else "?"}per_page=100&page={page}')[key] + out += got + if len(got) < 100: + return out + return out + + +# A truncated reply is http.client.IncompleteRead (an HTTPException), a garbled one a ValueError. +TROUBLE = (urllib.error.URLError, http.client.HTTPException, OSError, ValueError, KeyError, TypeError, + AttributeError) + + +def stamp(text): + try: + return calendar.timegm(time.strptime(text, '%Y-%m-%dT%H:%M:%SZ')) + except (TypeError, ValueError): + return None + + +def login(x): + return (x or {}).get('login') + + +# ---------------------------------------------------------------- the rules + +def untrusted(cfg, run): + """Why GitHub's record of a run is not one this script may re-run, or None.""" + repo = cfg['repository'] + if (run.get('repository') or {}).get('full_name') != repo: + return f'of another repository ({(run.get("repository") or {}).get("full_name")!r})' + if os.path.basename((run.get('path') or '').split('@')[0]) != cfg['workflow']: + return f'another workflow ({run.get("path")!r})' + if run.get('event') not in cfg['events']: + return f'event {run.get("event")!r}' + if login(run.get('actor')) not in cfg['actors']: + return f'started by {login(run.get("actor"))!r}' + if run.get('event') == 'push' and run.get('head_branch') not in cfg['push_branches']: + return f'a push to {run.get("head_branch")!r}' + if run.get('event') == 'pull_request' and (run.get('head_repository') or {}).get('full_name') != repo: + return 'a pull request from another repository' + return None + + +def note_of(annotations): + """The job's pressure note (fork-ci-pressure.py) as a dict, or (None, why).""" + notes = [a for a in annotations if a.get('title') == NOTE_TITLE] + if not notes: + return None, 'no pressure note (the job had no probe, or the note was dropped)' + if len(notes) > 1: + return None, f'{len(notes)} pressure notes, not one' + msg = notes[0].get('message') or '' + if not msg.startswith(NOTE_HEAD): + return None, 'a garbled pressure note' + try: + note = json.loads(msg[len(NOTE_HEAD):]) + except ValueError: + return None, 'a garbled pressure note' + if not (isinstance(note, dict) and isinstance(note.get('host'), dict) and isinstance(note.get('cgroups'), list) + and all(isinstance(c, dict) for c in note['cgroups'])): + return None, 'a garbled pressure note' + return note, None + + +def at_least(value, bound): + return number(value) and value >= bound + + +def stalled(cfg, note): + """Evidence that the machine or the job's box was stalled for memory, or None.""" + st, host = cfg['stalled'], note['host'] + if at_least(host.get('full_share'), st['full_share_percent']): + return f'the machine stalled (full) {host["full_share"]}% of the job' + for c in note['cgroups']: + if at_least(c.get('full_share'), st['full_share_percent']): + return f'box {c.get("path")} stalled (full) {c["full_share"]}% of the job' + if at_least(host.get('full_avg300'), st['full_avg300_percent']): + return f'the machine\'s 5-minute full stall average was {host["full_avg300"]}% at the end' + return None + + +def memory_full(cfg, note): + """Evidence that the job's box (or the machine) was full, or None.""" + mf, host = cfg['memory_full'], note['host'] + if at_least(host.get('oom_kill'), mf['oom_kills']): + return f'the kernel OOM-killed {host["oom_kill"]} process(es) on the machine during the job' + for c in note['cgroups']: + where = f'box {c.get("path")} (max {c.get("max_mb")} MB, high {c.get("high_mb")} MB)' + if at_least(c.get('oom_kill'), mf['oom_kills']): + return f'{where}: {c["oom_kill"]} OOM kill(s) during the job' + if at_least(c.get('max'), mf['max_events']): + return f'{where}: hit its memory.max {c["max"]} time(s) during the job' + share = max([s for s in (c.get('full_share'), host.get('full_share')) if number(s)], default=None) + if at_least(c.get('high'), 1) and at_least(share, mf['high_events_with_full_share_percent']): + return f'{where}: throttled at memory.high {c["high"]} time(s), stalled (full) {share}% of the job' + return None + + +def numbers(note): + """The note's numbers, for the log.""" + h = note['host'] + out = [f'machine: full {h.get("full_share")}% some {h.get("some_share")}% of {note.get("seconds")} s, ' + f'full avg10/60/300 {h.get("full_avg10")}/{h.get("full_avg60")}/{h.get("full_avg300")}%, ' + f'available {h.get("available_mb")} MB, swap used {h.get("swap_used_mb")} MB, OOM kills {h.get("oom_kill")}'] + out += [f'box {c.get("path")}: max {c.get("max_mb")} MB high {c.get("high_mb")} MB use {c.get("current_mb")} MB, ' + f'events high {c.get("high")} max {c.get("max")} oom_kill {c.get("oom_kill")}, full {c.get("full_share")}%' + for c in note['cgroups']] + return '; '.join(out) + + +def matches(patterns, messages): + for m in messages: + for p in patterns: + if re.search(p, m): + return m + return None + + +def job_sign(cfg, annotations): + """(sign, evidence) when a failed job shows an infrastructure sign, else (None, why not).""" + messages = [str(a.get('message') or '') for a in annotations if a.get('title') != NOTE_TITLE] + hit = matches(cfg['signs']['lost'], messages) + if hit: + return 'lost', f'GitHub: "{hit.strip()[:200]}"' + note, why = note_of(annotations) + full = memory_full(cfg, note) if note else None + if full: + return 'memory full', f'{full} ({numbers(note)})' + hit = matches(cfg['signs']['limit'], messages) + if hit and note: + stall = stalled(cfg, note) + if stall: + return 'limit while stalled', f'GitHub: "{hit.strip()[:200]}"; {stall} ({numbers(note)})' + return None, f'ran past its time limit, but the machine was not stalled ({numbers(note)})' + if hit: + return None, f'ran past its time limit, and {why}: cannot tell it was stalled' + return None, 'no infrastructure sign (a failure of its own, or a cancel)' + (f'; {why}' if not note else '') + + +def annotations_of(gh, job): + m = CHECK_RUN.search(job.get('check_run_url') or '') + got = gh.call(f'check-runs/{int(m[1]) if m else int(job["id"])}/annotations?per_page=100') + if not isinstance(got, list): + raise ValueError('annotations: not a list') + return got + + +def current(gh, run): + """Is the run's commit still the head of its branch? A newer push makes an old run moot.""" + try: + branch = gh.call(f'branches/{urllib.parse.quote(run.get("head_branch") or "", safe="")}') + except urllib.error.HTTPError as e: + if e.code == 404: + return False # the branch is gone (merged and deleted) + raise + return (branch.get('commit') or {}).get('sha') == run.get('head_sha') + + +def decide(cfg, gh, run, env): + """('quiet', None) for a run that is no candidate, ('leave', why), ('report', evidence) or ('rerun', evidence).""" + rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) + if run.get('status') != 'completed' or run.get('conclusion') not in RED: + return 'quiet', None + why = untrusted(cfg, run) + if why: + return 'leave', f'not a trusted run: {why}' + by = login(run.get('triggering_actor')) + if by not in cfg['actors']: + return 'leave', (f'attempt {attempt} was triggered by {by!r} (an automatic re-run): a repeat red stays red' + if attempt > 1 else f'triggered by {by!r}: not a trusted run') + for n in range(attempt - 1, 0, -1): + by = login(gh.call(f'actions/runs/{rid}/attempts/{n}').get('triggering_actor')) + if by not in cfg['actors']: + return 'leave', f'attempt {n} was triggered by {by!r} (an automatic re-run): never twice' + jobs = gh.pages(f'actions/runs/{rid}/jobs?filter=latest', 'jobs') + red = [j for j in jobs if j.get('conclusion') in RED and j.get('name') not in cfg['followers']] + if not red: + return 'leave', 'no failed job of its own (only followers failed, or nothing did)' + evidence = [] + for j in red: + sign, why = job_sign(cfg, annotations_of(gh, j)) + if not sign: + return 'leave', f'{j.get("name")} ended {j.get("conclusion")}: {why}' + evidence.append(f'{j.get("name")} ({j.get("conclusion")}): {sign}: {why}') + if not current(gh, run): + return 'leave', f'{run.get("head_branch")!r} has moved on or is gone; it qualified: ' + ' | '.join(evidence) + skip = cfg['skip_when'] + if (env.get(skip['variable']) or '').strip() in skip['is']: + return 'report', ' | '.join(evidence) + return 'rerun', ' | '.join(evidence) + + +def rerun(gh, run): + """GitHub's own re-run of failed jobs (and the jobs after them), once: (True, how) when this pass asked for it.""" + rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) + fresh = gh.call(f'actions/runs/{rid}') + if int(fresh.get('run_attempt') or 1) != attempt or fresh.get('status') != 'completed': + return False, f'already re-run or running again (attempt {fresh.get("run_attempt")}, {fresh.get("status")})' + try: + gh.call(f'actions/runs/{rid}/rerun-failed-jobs', method='POST') + except urllib.error.HTTPError as e: + again = gh.call(f'actions/runs/{rid}') + if int(again.get('run_attempt') or 1) != attempt or again.get('status') != 'completed': + return False, f'already re-run (GitHub refused a second request: HTTP {e.code})' + raise + return True, f'attempt {attempt + 1} asked for' + + +def summary(env, lines): + path = env.get('GITHUB_STEP_SUMMARY') + if path and lines: + with open(path, 'a', encoding='utf-8') as f: + f.write(''.join(f'{line}\n' for line in lines)) + + +def cell(text): + return str(text).replace('|', '\\|').replace('\n', ' ') + + +def sweep(cfg, env, gh, now=time.time, run_id=None, out=None): + if run_id: + runs = [gh.call(f'actions/runs/{int(run_id)}')] + else: + since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(now() - cfg['window_hours'] * 3600)) + runs = gh.pages(f'actions/workflows/{cfg["workflow"]}/runs?created=%3E%3D{since}', 'workflow_runs', most=3) + recent = now() - cfg['catch_up_minutes'] * 60 + rows, trouble, asked = [], 0, 0 + for run in runs: + rid = run.get('id') + try: + if not run_id and (stamp(run.get('updated_at')) or 0) < recent: + continue # ended before the catch-up window: too old to re-run now + what, text = decide(cfg, gh, run, env) + where = f'run {rid} ({run.get("event")} {run.get("head_branch")} {str(run.get("head_sha"))[:12]})' + if what == 'quiet': + continue + if what == 'leave': + say(f'{where}: left red: {text}', out) + rows.append(f'| {rid} | left red | {cell(text)} |') + continue + if what == 'report': + skip = cfg['skip_when'] + say(f'::error title=fork-ci re-run::{where} qualifies for its one automatic re-run, but ' + f'{skip["variable"]}={env.get(skip["variable"])}: not re-run; re-run it by hand ' + f'(.github/fork-ci-rerun.md). {text}', out) + rows.append(f'| {rid} | not re-run ({skip["variable"]}={cell(env.get(skip["variable"]))}) ' + f'| {cell(text)} |') + continue + done, how = rerun(gh, run) + if done: + asked += 1 + say(f'::warning title=fork-ci re-run::{where} re-run once ({how}), infrastructure: {text}', out) + rows.append(f'| {rid} | **re-run** ({cell(how)}) | {cell(text)} |') + else: + say(f'{where}: {how}', out) + rows.append(f'| {rid} | {cell(how)} | {cell(text)} |') + except TROUBLE as e: + trouble += 1 + say(f'::warning::fork-ci re-run: run {rid}: {e.__class__.__name__}: {e}; the next pass tries again', out) + rows.append(f'| {rid} | not read | {cell(e.__class__.__name__)}: {cell(e)} |') + bot = [r for r in runs if int(r.get('run_attempt') or 1) > 1 and login(r.get('triggering_actor')) + not in cfg['actors']] + count = (f'{asked} run(s) re-run by this pass; {len(bot)} run(s) whose latest attempt was an automatic re-run ' + f'(this script\'s or the fleet sweep\'s) in the last {cfg["window_hours"]} h' + + (': ' + ', '.join(str(r.get('id')) for r in bot) if bot else '')) + say(f'fork-ci re-run: {count}', out) + lines = ['### fork-ci re-run', '', count, ''] + if rows: + lines += ['| Run | Outcome | Why |', '| --- | --- | --- |'] + rows + summary(env, lines) + return 1 if trouble else 0 + + +def main(argv=None, env=os.environ): + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument('verb', choices=('sweep', 'check')) + p.add_argument('--run-id', type=int) + p.add_argument('--config', default=CONFIG) + a = p.parse_args(argv) + cfg = load(a.config) + if a.verb == 'check': + say(f'{a.config}: ok') + return 0 + return sweep(cfg, env, GitHub(env.get('GITHUB_TOKEN'), cfg['repository']), run_id=a.run_id) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/fork-ci-rerun.test.py b/.github/scripts/fork-ci-rerun.test.py new file mode 100644 index 0000000000000..876287039c2f7 --- /dev/null +++ b/.github/scripts/fork-ci-rerun.test.py @@ -0,0 +1,559 @@ +#!/usr/bin/env python3 +"""Tests of fork-ci-rerun.py (the one automatic re-run of an infrastructure failure), with a fake GitHub, and of its +wiring in fork-ci-rerun.yml. Run: python3 .""" +import copy, io, json, os, re, subprocess, sys, tempfile, time, unittest, urllib.error +import importlib.util + +sys.dont_write_bytecode = True # importing the script must not leave __pycache__ in the tree + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SCRIPT = os.path.join(HERE, 'fork-ci-rerun.py') +_spec = importlib.util.spec_from_file_location('fork_ci_rerun', SCRIPT) +cr = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cr) +_spec = importlib.util.spec_from_file_location('fork_ci_pressure', os.path.join(HERE, 'fork-ci-pressure.py')) +probe = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(probe) + +CFG = cr.load() +REPO = CFG['repository'] +OWNER = CFG['actors'][0] +BRANCH = CFG['push_branches'][0] +FOLLOWER = CFG['followers'][0] +SKIP_VAR, SKIP_VALUE = CFG['skip_when']['variable'], CFG['skip_when']['is'][0] +BOT = 'github-actions[bot]' +T0 = 1_800_000_000 +SHA = '0123456789abcdef0123456789abcdef01234567' +NEWER = 'fedcba9876543210fedcba9876543210fedcba98' + +# GitHub's own messages, as its annotations carry them. +LOST = ('The self-hosted runner: fleet-vm-1 lost communication with the server. Verify the machine is running and ' + 'has a healthy network connection. Anything in your workflow that terminates the runner process, starves it ' + 'for CPU/Memory, or blocks its network access can cause this error.') +SHUTDOWN = 'The runner has received a shutdown signal. This can happen when the runner service is stopped.' +JOB_LIMIT = 'The job has exceeded the maximum execution time of 15m0s' +STEP_LIMIT = "The action 'Test' has timed out after 10 minutes." +CANCELED = 'The operation was canceled.' +EXIT_1 = 'Process completed with exit code 1.' +EXIT_137 = 'Process completed with exit code 137.' + + +def iso(t): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t)) + + +def ann(message, level='failure', title=''): + return {'message': message, 'annotation_level': level, 'title': title} + + +def note(host=None, cgroups=None, status='failure', seconds=600): + """A pressure note as fork-ci-pressure.py writes it, decoded as GitHub hands it back.""" + h = {'full_share': 0.0, 'some_share': 0.5, 'full_avg10': 0.0, 'full_avg60': 0.0, 'full_avg300': 0.0, + 'some_avg60': 0.1, 'some_avg300': 0.1, 'available_mb': 9000, 'swap_used_mb': 0, 'oom_kill': 0} + h.update(host or {}) + boxes = cgroups if cgroups is not None else [box()] + body = json.dumps({'status': status, 'seconds': seconds, 'host': h, 'cgroups': boxes}, separators=(',', ':')) + return ann(cr.NOTE_HEAD + body, level='notice', title=cr.NOTE_TITLE) + + +def box(**kw): + b = {'path': '/fleet.slice/job-1.service', 'high_mb': 6144, 'max_mb': 8192, 'current_mb': 3000, + 'high': 0, 'max': 0, 'oom': 0, 'oom_kill': 0, 'full_share': 0.0} + b.update(kw) + return b + + +CALM = note() + + +def run(rid=1, attempt=1, conclusion='failure', status='completed', event='push', branch=BRANCH, actor=OWNER, + by=OWNER, head_repo=REPO, updated=T0 - 120, workflow='fork-tests.yml'): + return {'id': rid, 'run_attempt': attempt, 'status': status, 'conclusion': conclusion, 'event': event, + 'head_branch': branch, 'head_sha': SHA, 'path': f'.github/workflows/{workflow}', + 'repository': {'full_name': REPO}, 'head_repository': {'full_name': head_repo}, + 'actor': {'login': actor}, 'triggering_actor': {'login': by}, 'updated_at': iso(updated)} + + +def job(jid, name='Test (1)', conclusion='failure'): + return {'id': jid, 'name': name, 'conclusion': conclusion, 'status': 'completed', + 'check_run_url': f'https://api.example/repos/{REPO}/check-runs/{jid}'} + + +class Fake: + """GitHub's REST API as fork-ci-rerun.py uses it; a re-run makes a new attempt triggered by github-actions.""" + + def __init__(self, runs, jobs=None, annotations=None, heads=None): + self.runs = {r['id']: r for r in runs} + self.jobs = jobs or {} + self.ann = annotations or {} + self.heads = heads if heads is not None else {BRANCH: SHA, 'feature': SHA} + self.attempts = {} + self.calls = [] + self.post_fails = None # an exception to raise for the next POST + self.lost_reply = False # the POST reaches GitHub, its reply never comes back + + def call(self, path, method='GET', data=None): + self.calls.append((method, path)) + if method == 'POST': + m = re.fullmatch(r'actions/runs/(\d+)/rerun-failed-jobs', path) + if not m: + raise AssertionError(f'unexpected POST {path}') + r = self.runs[int(m[1])] + if self.post_fails: + e, self.post_fails = self.post_fails, None + raise e() if callable(e) and not isinstance(e, BaseException) else e + if r['status'] != 'completed': + raise urllib.error.HTTPError(path, 403, 'This workflow is already running', {}, None) + self.attempts[(r['id'], r['run_attempt'])] = copy.deepcopy(r) + r.update(run_attempt=r['run_attempt'] + 1, status='queued', conclusion=None, + triggering_actor={'login': BOT}) + if self.lost_reply: + raise urllib.error.URLError('connection reset') + return {} + m = re.fullmatch(r'actions/runs/(\d+)', path) + if m: + return copy.deepcopy(self.runs[int(m[1])]) + m = re.fullmatch(r'actions/runs/(\d+)/attempts/(\d+)', path) + if m: + got = self.attempts.get((int(m[1]), int(m[2]))) + if got is None: + got = dict(self.runs[int(m[1])], run_attempt=int(m[2]), triggering_actor={'login': OWNER}) + return copy.deepcopy(got) + m = re.fullmatch(r'check-runs/(\d+)/annotations\?per_page=100', path) + if m: + return copy.deepcopy(self.ann.get(int(m[1]), [])) + m = re.fullmatch(r'branches/(.+)', path) + if m: + if m[1] not in self.heads: + raise urllib.error.HTTPError(path, 404, 'Branch not found', {}, None) + return {'commit': {'sha': self.heads[m[1]]}} + raise AssertionError(f'unexpected GET {path}') + + def pages(self, path, key, most=10): + self.calls.append(('GET', path)) + if path.startswith(f'actions/workflows/{CFG["workflow"]}/runs?created=%3E%3D'): + return [copy.deepcopy(r) for r in self.runs.values()] + m = re.fullmatch(r'actions/runs/(\d+)/jobs\?filter=latest', path) + if m: + return copy.deepcopy(self.jobs.get(int(m[1]), [])) + raise AssertionError(f'unexpected pages {path}') + + def posts(self): + return [p for m, p in self.calls if m == 'POST'] + + +def sweep(gh, env=None, run_id=None, cfg=CFG): + out = io.StringIO() + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, 'summary.md') + code = cr.sweep(cfg, dict(env or {}, GITHUB_STEP_SUMMARY=path), gh, now=lambda: T0, run_id=run_id, out=out) + text = '' + if os.path.exists(path): + with open(path, encoding='utf-8') as f: + text = f.read() + return code, out.getvalue(), text + + +def one(annotations, job_conclusion='failure', **kw): + """A failed run whose one failed job (id 11) carries these annotations, plus a passed job and the follower.""" + jobs = [job(11, conclusion=job_conclusion), job(12, 'Lint', 'success'), job(13, FOLLOWER, 'failure')] + return Fake([run(**kw)], {1: jobs}, {11: annotations, 13: [ann(EXIT_1)]}) + + +class Signs(unittest.TestCase): + def assertReran(self, gh, out): + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + self.assertIn('::warning title=fork-ci re-run::run 1', out) + + def test_lost_runner_is_rerun_once_with_the_message_in_the_log(self): + for message in (LOST, SHUTDOWN, 'The hosted runner: GitHub Actions 7 lost communication with the server.'): + gh = one([ann(message), ann(CANCELED)]) + code, out, text = sweep(gh) + self.assertEqual(code, 0, out) + self.assertReran(gh, out) + self.assertIn('lost', out) + self.assertIn('**re-run**', text) + + def test_lost_runner_needs_no_pressure_note_an_old_run_without_the_new_lines(self): + gh = one([ann(LOST)]) # the job never reached its end step: no note + self.assertReran(gh, sweep(gh)[1]) + + def test_a_full_box_is_rerun_with_its_numbers_in_the_log(self): + cases = { + 'box OOM kill': note(cgroups=[box(oom_kill=1)]), + 'box memory.max hits': note(cgroups=[box(max=3)]), + 'machine OOM kill': note(host={'oom_kill': 2}), + 'memory.high throttling while stalled': note(host={'full_share': 25.0}, cgroups=[box(high=40)]), + 'memory.high throttling, the box stalled': note(cgroups=[box(high=40, full_share=30.0)]), + } + for label, n in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), n]) + code, out, text = sweep(gh) + self.assertReran(gh, out) + self.assertIn('memory full', out) + self.assertRegex(out, r'machine: full [0-9.]+% some [0-9.]+% of 600 s') + self.assertIn('box /fleet.slice/job-1.service: max 8192 MB high 6144 MB', out) + self.assertIn('memory full', text) + + def test_memory_high_events_without_a_stall_are_no_sign(self): + gh = one([ann(EXIT_1), note(host={'full_share': 2.0}, cgroups=[box(high=40, full_share=3.0)])]) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), [], out) + self.assertIn('no infrastructure sign', out) + + def test_time_limit_while_stalled_is_rerun(self): + for message in (JOB_LIMIT, STEP_LIMIT): + for stall in ({'full_share': 12.0}, {'full_avg300': 15.0}): + with self.subTest(message=message, stall=stall): + gh = one([ann(message), ann(CANCELED), note(host=stall, status='cancelled')], 'cancelled', + conclusion='cancelled') + code, out, _ = sweep(gh) + self.assertReran(gh, out) + self.assertIn('limit while stalled', out) + gh = one([ann(JOB_LIMIT), note(cgroups=[box(full_share=50.0)], status='cancelled')], 'cancelled') + self.assertReran(gh, sweep(gh)[1]) + + def test_time_limit_on_a_calm_machine_is_left_red(self): + gh = one([ann(JOB_LIMIT), ann(CANCELED), CALM], 'cancelled') + code, out, text = sweep(gh) + self.assertEqual(gh.posts(), [], out) + self.assertIn('not stalled', out) + self.assertIn('left red', text) + + def test_time_limit_without_a_note_is_left_red_fail_closed(self): + gh = one([ann(JOB_LIMIT), ann(CANCELED)], 'cancelled') # an old run, or the note was dropped + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), [], out) + self.assertIn('cannot tell it was stalled', out) + + def test_a_test_failure_is_never_rerun(self): + gh = one([ann(EXIT_1), CALM]) + code, out, text = sweep(gh) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('Test (1) ended failure: no infrastructure sign', out) + self.assertIn('| 1 | left red |', text) + + def test_a_job_with_no_readable_signs_is_left_red(self): + garbled = dict(CALM, message=cr.NOTE_HEAD + '{"host": ') + cases = {'no annotations at all': [], 'a garbled note': [ann(EXIT_1), garbled], + 'a note without its head': [ann(EXIT_1), dict(CALM, message='{"host":{},"cgroups":[]}')], + 'a note of the wrong shape': [ann(EXIT_1), dict(CALM, message=cr.NOTE_HEAD + '[1, 2]')], + 'two notes (one forged)': [ann(EXIT_137), CALM, note(cgroups=[box(oom_kill=5)])], + 'numbers that are not numbers': [ann(EXIT_137), note(host={'oom_kill': '9'}, + cgroups=[box(oom_kill=True, max='x')])], + 'a cancel by a person': [ann(CANCELED)]} + for label, anns in cases.items(): + with self.subTest(label): + gh = one(anns) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('left red', out) + + def test_one_job_without_a_sign_leaves_the_whole_run_red(self): + jobs = [job(11), job(12, 'Test (2)'), job(13, FOLLOWER)] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_1), CALM]}) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), [], out) + self.assertIn('Test (2) ended failure', out) + + def test_every_failed_job_with_a_sign_is_rerun_together(self): + jobs = [job(11), job(12, 'Build and Typecheck'), job(13, FOLLOWER)] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_137), note(cgroups=[box(oom_kill=1)])]}) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + self.assertIn('Test (1) (failure): lost', out) + self.assertIn('Build and Typecheck (failure): memory full', out) + + def test_only_a_follower_failed_is_left_red(self): + gh = Fake([run()], {1: [job(12, 'Lint', 'success'), job(13, FOLLOWER)]}, {13: [ann(LOST)]}) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), [], out) + self.assertIn('no failed job of its own', out) + + def test_a_job_skipped_after_a_failed_one_needs_no_sign(self): + jobs = [job(11), job(12, 'Lint', 'skipped'), job(13, FOLLOWER)] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)]}) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + + +class NeverTwice(unittest.TestCase): + def test_a_second_failure_after_the_automatic_rerun_stays_red(self): + gh = one([ann(LOST)], attempt=2, by=BOT) + code, out, text = sweep(gh) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('a repeat red stays red', out) + self.assertIn('1 run(s) whose latest attempt was an automatic re-run', out) + self.assertIn('1 run(s) whose latest attempt was an automatic re-run', text) + + def test_a_person_rerunning_after_the_automatic_rerun_gets_no_second_one(self): + gh = one([ann(LOST)], attempt=3, by=OWNER) + gh.attempts[(1, 2)] = run(attempt=2, by=BOT) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), [], out) + self.assertIn('attempt 2 was triggered by', out) + self.assertIn('never twice', out) + + def test_a_person_rerun_without_an_automatic_one_may_get_it(self): + gh = one([ann(LOST)], attempt=2, by=OWNER) # attempt 1 by the owner too (the fake's default) + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + + def test_the_watcher_runs_again_on_the_run_it_reran(self): + gh = one([ann(LOST)]) + sweep(gh) + self.assertEqual(len(gh.posts()), 1) + code, out, _ = sweep(gh) # the re-run is queued: not completed, nothing to do + self.assertEqual((code, len(gh.posts())), (0, 1), out) + gh.runs[1].update(status='completed', conclusion='failure') # and it failed again, the same way + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts())), (0, 1), out) + self.assertIn('a repeat red stays red', out) + + def test_the_request_sent_twice_github_refuses_the_second(self): + gh = one([ann(LOST)]) + first = copy.deepcopy(gh.runs[1]) + sweep(gh) + # A second watcher (or the fleet sweep) read the run before the first re-ran it and asks again. + done, how = cr.rerun(gh, first) + self.assertFalse(done) + self.assertIn('already re-run', how) + self.assertEqual(len(gh.posts()), 1, 'the second request is not even sent: the run is read again first') + + def test_github_refusing_a_rerun_that_already_started_is_no_trouble(self): + gh = one([ann(LOST)]) + # Another watcher re-ran it between this pass's last read and its request. + def race(): + gh.runs[1].update(run_attempt=2, status='queued', triggering_actor={'login': BOT}) + return urllib.error.HTTPError('x', 403, 'This workflow is already running', {}, None) + gh.post_fails = race + code, out, text = sweep(gh) + self.assertEqual(code, 0, out) + self.assertIn('already re-run (GitHub refused a second request: HTTP 403)', out) + + def test_a_refusal_with_the_run_unchanged_is_trouble_and_tried_again(self): + gh = one([ann(LOST)]) + gh.post_fails = urllib.error.HTTPError('x', 500, 'Server Error', {}, None) + code, out, text = sweep(gh) + self.assertEqual(code, 1, out) + self.assertIn('the next pass tries again', out) + self.assertIn('not read', text) + code, out, _ = sweep(gh) # the next pass + self.assertEqual(code, 0, out) + self.assertEqual(len(gh.posts()), 2) + self.assertEqual(gh.runs[1]['run_attempt'], 2) + + def test_the_watcher_stopped_half_way_the_request_never_reached_github(self): + gh = one([ann(LOST)]) + gh.post_fails = urllib.error.URLError('timed out') + code, out, _ = sweep(gh) + self.assertEqual((code, gh.runs[1]['run_attempt']), (1, 1), out) + code, out, _ = sweep(gh) # the next pass finishes it, once + self.assertEqual((code, gh.runs[1]['run_attempt']), (0, 2), out) + self.assertEqual(len(gh.posts()), 2) + sweep(gh) + self.assertEqual(len(gh.posts()), 2) + + def test_the_watcher_stopped_half_way_the_rerun_was_asked_but_not_started(self): + gh = one([ann(LOST)]) + gh.lost_reply = True # GitHub made attempt 2; the watcher never heard back + code, out, _ = sweep(gh) + self.assertEqual(code, 1, out) + gh.lost_reply = False + for _ in range(2): # queued, never started: the next passes leave it alone + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts())), (0, 1), out) + gh.runs[1].update(status='completed', conclusion='cancelled') # it never ran (cancelled while queued) + code, out, _ = sweep(gh) + self.assertEqual(len(gh.posts()), 1, out) + self.assertIn('a repeat red stays red', out) + + +class Scope(unittest.TestCase): + def test_untrusted_runs_are_left_alone(self): + cases = {'a stranger': dict(actor='someone-else', by='someone-else'), + 'triggered by a stranger': dict(by='someone-else'), + 'a pull request from a fork': dict(event='pull_request', branch='feature', + head_repo='someone-else/t3code'), + 'a push to another branch': dict(branch='feature'), + 'another event': dict(event='workflow_dispatch'), + 'another workflow': dict(workflow='other.yml')} + for label, kw in cases.items(): + with self.subTest(label): + gh = one([ann(LOST)], **kw) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('not a trusted run', out) + self.assertFalse([p for _, p in gh.calls if '/jobs' in p], 'an untrusted run costs no job read') + + def test_an_own_pull_request_is_rerun(self): + gh = one([ann(LOST)], event='pull_request', branch='feature') + code, out, _ = sweep(gh) + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + + def test_a_branch_that_moved_on_or_is_gone_is_not_rerun(self): + for heads in ({BRANCH: NEWER}, {}): + with self.subTest(heads=heads): + gh = one([ann(LOST)]) + gh.heads = heads + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('has moved on or is gone', out) + + def test_runs_that_passed_are_still_running_or_ended_long_ago_cost_nothing(self): + gh = Fake([run(1, conclusion='success'), run(2, status='in_progress', conclusion=None), + run(3, updated=T0 - CFG['catch_up_minutes'] * 60 - 1)], {}, {}) + code, out, text = sweep(gh) + self.assertEqual(code, 0, out) + self.assertEqual([p for _, p in gh.calls if not p.startswith('actions/workflows/')], []) + self.assertIn('0 run(s) re-run by this pass', text) + + def test_an_empty_list_of_runs(self): + code, out, text = sweep(Fake([])) + self.assertEqual(code, 0, out) + self.assertIn('0 run(s) re-run by this pass; 0 run(s)', out) + self.assertIn('### fork-ci re-run', text) + + def test_one_run_by_id_ignores_the_catch_up_window(self): + gh = one([ann(LOST)], updated=T0 - 5 * 3600) + code, out, _ = sweep(gh, run_id=1) + self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + self.assertFalse([p for _, p in gh.calls if p.startswith('actions/workflows/')]) + + def test_fleet_mode_reports_and_does_not_rerun(self): + gh = one([ann(LOST)]) + code, out, text = sweep(gh, env={SKIP_VAR: SKIP_VALUE}) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('::error title=fork-ci re-run::', out) + self.assertIn('re-run it by hand', out) + self.assertIn(f'not re-run ({SKIP_VAR}={SKIP_VALUE})', text) + gh = one([ann(LOST)]) + sweep(gh, env={SKIP_VAR: 'auto'}) + self.assertEqual(len(gh.posts()), 1) + + def test_a_github_error_on_one_run_does_not_stop_the_others(self): + gh = Fake([run(1), run(2)], {2: [job(21)]}, {21: [ann(LOST)]}) + real = gh.pages + def pages(path, key, most=10): + if path.startswith('actions/runs/1/'): + raise urllib.error.URLError('reset') + return real(path, key, most) + gh.pages = pages + code, out, _ = sweep(gh) + self.assertEqual(code, 1, out) + self.assertEqual(gh.posts(), ['actions/runs/2/rerun-failed-jobs']) + + def test_a_pipe_in_a_message_keeps_the_summary_table_whole(self): + gh = one([ann('The hosted runner: a | b lost communication with the server.'), CALM]) + _, _, text = sweep(gh) + self.assertIn('a \\| b', text) + row = [l for l in text.splitlines() if l.startswith('| 1 |')][0] + self.assertEqual(row.count(' | '), 2, row) + + +class Config(unittest.TestCase): + def test_the_real_config_is_valid_and_names_the_rules_parts(self): + self.assertEqual(cr.check_config(CFG), []) + r = subprocess.run([sys.executable, '-I', SCRIPT, 'check'], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + + def test_github_messages_match_their_sign_and_nothing_else_does(self): + lost, limit = CFG['signs']['lost'], CFG['signs']['limit'] + for m in (LOST, SHUTDOWN): + self.assertTrue(cr.matches(lost, [m]) and not cr.matches(limit, [m]), m) + for m in (JOB_LIMIT, STEP_LIMIT): + self.assertTrue(cr.matches(limit, [m]) and not cr.matches(lost, [m]), m) + for m in (EXIT_1, EXIT_137, CANCELED, 'Error: the test timed out after 5000 ms'): + self.assertFalse(cr.matches(lost + limit, [m]), m) + + def test_bad_configs_are_refused(self): + def bad(change): + cfg = copy.deepcopy(CFG) + change(cfg) + return cr.check_config(cfg) + cases = { + 'a bad pattern': lambda c: c['signs']['lost'].append('(['), + 'an unknown sign': lambda c: c['signs'].__setitem__('exit', ['137']), + 'no actors': lambda c: c.__setitem__('actors', []), + 'an event a fork can cause': lambda c: c['events'].append('workflow_run'), + 'a zero threshold': lambda c: c['stalled'].__setitem__('full_share_percent', 0), + 'a missing threshold': lambda c: c['memory_full'].pop('max_events'), + 'a string threshold': lambda c: c['memory_full'].__setitem__('oom_kills', '1'), + 'an unknown key': lambda c: c.__setitem__('retries', 2), + 'no skip variable': lambda c: c['skip_when'].pop('variable'), + 'no window': lambda c: c.__setitem__('catch_up_minutes', -1), + } + for label, change in cases.items(): + with self.subTest(label): + self.assertTrue(bad(change), label) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + json.dump(dict(CFG, retries=2), f) + try: + r = subprocess.run([sys.executable, '-I', SCRIPT, 'check', '--config', f.name], capture_output=True, + text=True) + self.assertNotEqual(r.returncode, 0) + self.assertIn("unknown key 'retries'", r.stderr) + finally: + os.unlink(f.name) + + def test_the_script_holds_no_fact_of_this_repository(self): + with open(SCRIPT, encoding='utf-8') as f: + text = f.read() + for fact in [REPO, OWNER, CFG['workflow'], FOLLOWER] + CFG['signs']['lost'] + CFG['signs']['limit']: + self.assertFalse(fact in text.replace('fork-ci-rerun', ''), f'the script names {fact!r}') + + +class ProbeToDecider(unittest.TestCase): + """The probe's notice, as GitHub decodes it, is the note the decider reads (one contract, both ends).""" + + def test_a_probe_notice_round_trips(self): + n = {'seconds': 300, 'host': {'full_share': 40.0, 'full_avg300': 1.0, 'oom_kill': 0}, + 'cgroups': [{'path': '/a%b\nc', 'oom_kill': 2, 'max': 0, 'high': 0, 'full_share': None}]} + out = io.StringIO() + body = json.dumps({'status': 'failure', **n}, separators=(',', ':'), sort_keys=True) + print(f'::notice title={probe.NOTE_TITLE}::{probe.command(probe.NOTE_HEAD + body)}', file=out) + line = out.getvalue().strip() + title, _, data = line[len('::notice '):].partition('::') + self.assertEqual(title, f'title={cr.NOTE_TITLE}') + self.assertNotIn('\n', data) + message = data.replace('%0A', '\n').replace('%0D', '\r').replace('%25', '%') # what the runner does + got, why = cr.note_of([ann(message, 'notice', cr.NOTE_TITLE)]) + self.assertIsNone(why) + self.assertEqual(got['cgroups'][0]['path'], '/a%b\nc') + self.assertEqual(cr.job_sign(CFG, [ann(EXIT_137), ann(message, 'notice', cr.NOTE_TITLE)])[0], 'memory full') + self.assertEqual((probe.NOTE_TITLE, probe.NOTE_HEAD), (cr.NOTE_TITLE, cr.NOTE_HEAD)) + + +class Wiring(unittest.TestCase): + def setUp(self): + with open(os.path.join(ROOT, '.github/workflows/fork-ci-rerun.yml'), encoding='utf-8') as f: + self.text = f.read() + + def test_the_workflow_runs_the_sweep_on_a_schedule_with_least_permissions(self): + import yaml + wf = yaml.safe_load(self.text) + self.assertEqual(set(wf[True]), {'schedule', 'workflow_dispatch'}) + self.assertEqual(wf['permissions'], {}) + sweep_job = wf['jobs']['sweep'] + self.assertEqual(sweep_job['permissions'], {'actions': 'write', 'checks': 'read', 'contents': 'read'}) + self.assertFalse(wf['concurrency']['cancel-in-progress'], 'a pass is never cut off by the next') + step = sweep_job['steps'][-1] + self.assertIn('python3 -I .github/scripts/fork-ci-rerun.py sweep', step['run']) + self.assertEqual(step['env']['CI_RUNNERS'], '${{ vars.CI_RUNNERS }}') + self.assertEqual(SKIP_VAR, 'CI_RUNNERS') + checkout = sweep_job['steps'][0]['with']['sparse-checkout'] + self.assertIn('/.github/fork-ci-rerun.json', checkout) + self.assertIn('/.github/scripts/fork-ci-rerun.py', checkout) + + def test_the_sweep_is_offset_from_the_fleet_sweep(self): + import yaml + ours = yaml.safe_load(self.text)[True]['schedule'][0]['cron'] + with open(os.path.join(ROOT, '.github/workflows/fork-fleet.yml'), encoding='utf-8') as f: + fleet = yaml.safe_load(f)[True]['schedule'][0]['cron'] + self.assertEqual((ours.split()[0], fleet.split()[0]), ('5-55/10', '*/10')) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/fork-ci-rerun.yml b/.github/workflows/fork-ci-rerun.yml new file mode 100644 index 0000000000000..5e91e8deb7148 --- /dev/null +++ b/.github/workflows/fork-ci-rerun.yml @@ -0,0 +1,61 @@ +name: Fork CI re-run + +# Fork only. A failed Fork tests run gets ONE automatic "re-run failed jobs" +# (GitHub's own), only when every failed job shows a known infrastructure sign: +# its runner lost communication, its box was full (OOM kills, memory.max hits, +# memory.high throttling while stalled), or it ran past its time limit while the +# machine was stalled. Never for a test failure (no sign: left red, fail closed) +# and never twice: the re-run is triggered by github-actions, and a run with an +# attempt by anyone but CONFIG's actors is left red. Every rule, and the numbers +# that decide it, are in .github/scripts/fork-ci-rerun.py and +# .github/fork-ci-rerun.json; each job's numbers come from its own +# fork-ci-pressure.py steps (fork-tests.yml). Each re-run is a warning +# annotation with its signs and numbers and a row in this job's summary, and +# every pass counts the automatic re-runs of the last 24 hours. +# On a schedule, not on workflow_run (fork-contracts.test.py refuses events a +# pull request from another fork can cause): nothing is kept between passes, +# each reads the runs of the last hour from GitHub, so a pass that stopped +# half-way is finished by the next. Offset from fork-fleet.yml's sweep, which +# re-runs a lost fleet runner's run too: whichever comes first re-runs it, the +# other finds an attempt by github-actions and leaves it. Runbook: +# .github/fork-ci-rerun.md. +on: + schedule: + - cron: "5-55/10 * * * *" + workflow_dispatch: + inputs: + run_id: + description: Only this Fork tests run (its id; empty for every run of the last hour) + required: false + type: string + +permissions: {} + +concurrency: + group: fork-ci-rerun + cancel-in-progress: false + +jobs: + sweep: + name: Sweep + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: write + checks: read + contents: read + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + sparse-checkout: | + /.github/fork-ci-rerun.json + /.github/scripts/fork-ci-rerun.py + sparse-checkout-cone-mode: false + + - name: Re-run a run that failed for infrastructure only, once + env: + CI_RUNNERS: ${{ vars.CI_RUNNERS }} + GITHUB_TOKEN: ${{ github.token }} + RUN_ID: ${{ inputs.run_id }} + run: python3 -I .github/scripts/fork-ci-rerun.py sweep ${RUN_ID:+--run-id "$RUN_ID"} diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index f25054adacdd0..95b4b9b3c78cf 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -746,6 +746,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + - name: Reject repository-owned PR assets run: | files="$(git ls-files .github/pr-assets)" @@ -800,6 +805,13 @@ jobs: - name: Check run: vp check + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + build: name: Build and Typecheck needs: changes @@ -818,6 +830,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + # Runs in the background while Vite+ installs; the step before the build waits for it. # Xvfb is for the desktop launch check below. - name: Start installing browser secret helper build libraries @@ -956,6 +973,13 @@ jobs: cat "$RUNNER_TEMP/typecheck.log" exit "$(cat "$RUNNER_TEMP/typecheck-status")" + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + # Everything except `t3` (apps/server) and the web app, which have their own jobs. # Upstream runs this on one large runner; one 4-CPU runner took 6 minutes. # Two runners split it (.github/scripts/fork-test-shards.cjs): each runs a @@ -982,6 +1006,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + # A shard with nothing to run says so in the summary and stops here. - name: Pick this shard's packages id: shard @@ -1086,6 +1115,13 @@ jobs: done exit "$status" + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + # Upstream runs this on one large runner; on a 4-CPU runner the 548 files # took 7.5 minutes, so each of the shards below takes a vitest slice. test_web: @@ -1113,6 +1149,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + - name: Setup Vite+ uses: ./.github/actions/setup-ci-deps with: @@ -1163,6 +1204,13 @@ jobs: working-directory: apps/web run: eval "$(jq -r .scripts.test package.json) --shard=$SHARD" + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + # apps/server runs its test files one at a time; shards spread them over # separate runners. As upstream, the shards run the change detector # themselves, so the steps follow ci.yml one for one; a shard the change @@ -1192,6 +1240,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + - name: Detect affected jobs id: changes if: inputs.ref == '' @@ -1311,6 +1364,13 @@ jobs: if-no-files-found: ignore retention-days: 30 + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + rust: name: Rust needs: changes @@ -1329,6 +1389,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + - name: Setup Rust uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: @@ -1347,6 +1412,13 @@ jobs: cargo test --locked --manifest-path "native/$crate/Cargo.toml" done + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + # Upstream runs this on macOS 26. GitHub-hosted macOS minutes are free on a # public repository; it runs only for native mobile sources or a workflow change. mobile_native_static_analysis: @@ -1398,6 +1470,11 @@ jobs: !/apps/mobile/modules/t3-terminal/Vendor/**/*.a sparse-checkout-cone-mode: false + # Memory pressure from here on, for the one automatic re-run of an infrastructure failure + # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). + - name: Note memory pressure at the start + run: python3 -I .github/scripts/fork-ci-pressure.py start + - name: Setup Vite+ uses: ./.github/actions/setup-ci-deps with: @@ -1407,6 +1484,13 @@ jobs: - name: Exercise release-only workflow steps run: node scripts/release-smoke.ts + # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. + - name: Note memory pressure at the end + if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} + env: + JOB_STATUS: ${{ job.status }} + run: python3 -I .github/scripts/fork-ci-pressure.py end + # The one check a ruleset requires. Fails when any job fails or is cancelled, # or is skipped without a reason: `changes` said its gates were not needed, # or (secret scan) it is a release run. The summary lists every job, the From 9ccfa18c07d6b37fa3dc384ffc1607e0783bf149 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 02:20:18 +0000 Subject: [PATCH 2/4] fork CI: name the pressure note fork-ci-pressure (#130) fork-owned-reads.test.py reads "fork-ci pressure" in the `!`-listed probe as the fork-owned fork-ci/ folder; the script's own name is no read. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-rerun.md | 4 ++-- .github/scripts/fork-ci-pressure.py | 10 +++++----- .github/scripts/fork-ci-pressure.test.py | 8 ++++---- .github/scripts/fork-ci-rerun.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/fork-ci-rerun.md b/.github/fork-ci-rerun.md index 01658ac4db025..245bbc87c10b6 100644 --- a/.github/fork-ci-rerun.md +++ b/.github/fork-ci-rerun.md @@ -18,7 +18,7 @@ and its fix first. The `end` step prints one line in every app job, passed or not: ``` -fork-ci pressure: job failure; 612 s since start; memory stall full 14.2% some 30.1% of the job; +fork-ci-pressure: job failure; 612 s since start; memory stall full 14.2% some 30.1% of the job; full avg10/60/300 22.0/18.4/12.9%; available 310 MB, swap used 1024 MB, OOM kills 1; cgroup /fleet.slice/fork-fleet-x.service: high None MB max None MB use 2900 MB, events high None max None oom_kill None, full 15.0%; ... ``` @@ -33,7 +33,7 @@ cgroup /fleet.slice/fork-fleet-x.service: high None MB max None MB use 2900 MB, `memory.events` (high, max, oom_kill) since `start` and its own stall share. A job that did not succeed also leaves the same numbers as one notice -annotation (`fork-ci pressure`), which the decision reads from GitHub after the +annotation (`fork-ci-pressure`), which the decision reads from GitHub after the run. `null` (or `None` in the line) is a number that could not be read: no start sample, a reboot in between, a file the runner cannot see. diff --git a/.github/scripts/fork-ci-pressure.py b/.github/scripts/fork-ci-pressure.py index c1c4861af3954..9b12d4f62a810 100644 --- a/.github/scripts/fork-ci-pressure.py +++ b/.github/scripts/fork-ci-pressure.py @@ -19,7 +19,7 @@ """ import json, os, sys, time -NOTE_TITLE = 'fork-ci pressure' +NOTE_TITLE = 'fork-ci-pressure' NOTE_HEAD = 'fork-ci-pressure v1 ' SAMPLE = 'fork-ci-pressure-start.json' EVENTS = ('high', 'max', 'oom', 'oom_kill') @@ -155,7 +155,7 @@ def numbers(start, end): def line(status, n): h = n['host'] - out = [f'fork-ci pressure: job {status}', f'{n["seconds"]} s since start' if n['seconds'] is not None + out = [f'fork-ci-pressure: job {status}', f'{n["seconds"]} s since start' if n['seconds'] is not None else 'no start sample (numbers since start unknown)', f'memory stall full {h["full_share"]}% some {h["some_share"]}% of the job', f'full avg10/60/300 {h["full_avg10"]}/{h["full_avg60"]}/{h["full_avg300"]}%', @@ -175,7 +175,7 @@ def main(argv, env=os.environ, out=sys.stdout, proc='/proc', root='/sys/fs/cgrou verb = argv[1] if len(argv) == 2 else '' temp = env.get('RUNNER_TEMP') or '' if verb not in ('start', 'end') or not temp: - print('::warning::fork-ci pressure: usage: fork-ci-pressure.py start|end, with RUNNER_TEMP set', file=out) + print('::warning::fork-ci-pressure: usage: fork-ci-pressure.py start|end, with RUNNER_TEMP set', file=out) return 0 path = os.path.join(temp, SAMPLE) try: @@ -183,7 +183,7 @@ def main(argv, env=os.environ, out=sys.stdout, proc='/proc', root='/sys/fs/cgrou if verb == 'start': with open(path, 'w', encoding='utf-8') as f: json.dump(now_sample, f) - print(f'fork-ci pressure: start sample saved ({len(now_sample["cgroups"])} cgroup(s))', file=out) + print(f'fork-ci-pressure: start sample saved ({len(now_sample["cgroups"])} cgroup(s))', file=out) return 0 try: start = json.loads(read(path) or 'null') @@ -196,7 +196,7 @@ def main(argv, env=os.environ, out=sys.stdout, proc='/proc', root='/sys/fs/cgrou note = json.dumps({'status': status, **n}, separators=(',', ':'), sort_keys=True) print(f'::notice title={NOTE_TITLE}::{command(NOTE_HEAD + note)}', file=out) except Exception as e: # never fail the job - print(f'::warning::fork-ci pressure: {verb}: {e.__class__.__name__}: {e}', file=out) + print(f'::warning::fork-ci-pressure: {verb}: {e.__class__.__name__}: {e}', file=out) return 0 diff --git a/.github/scripts/fork-ci-pressure.test.py b/.github/scripts/fork-ci-pressure.test.py index 996872ca3da94..1d0ea7cccc808 100644 --- a/.github/scripts/fork-ci-pressure.test.py +++ b/.github/scripts/fork-ci-pressure.test.py @@ -95,7 +95,7 @@ def test_a_failed_job_prints_its_numbers_and_one_note(self): self.stall() code, out = run(self.m, 'end', self.temp, 'failure', t=1600.0) self.assertEqual(code, 0) - self.assertIn('fork-ci pressure: job failure; 600 s since start; memory stall full 10.0% some 20.0%', out) + self.assertIn('fork-ci-pressure: job failure; 600 s since start; memory stall full 10.0% some 20.0%', out) self.assertIn('full avg10/60/300 30.0/0.0/12.5%', out) self.assertIn('available 8000 MB, swap used 1024 MB, OOM kills 2', out) self.assertIn('cgroup /fleet.slice: high None MB max 8192 MB', out) @@ -113,7 +113,7 @@ def test_a_passing_job_prints_its_numbers_but_no_note(self): run(self.m, 'start', self.temp) code, out = run(self.m, 'end', self.temp, 'success', t=1600.0) self.assertEqual(code, 0) - self.assertIn('fork-ci pressure: job success', out) + self.assertIn('fork-ci-pressure: job success', out) self.assertIsNone(the_note(out)) def test_a_cancelled_job_and_an_unknown_status_leave_a_note(self): @@ -183,11 +183,11 @@ def test_it_never_fails_the_job(self): if os.geteuid() != 0: code, text = run(self.m, 'start', self.temp) self.assertEqual(code, 0) - self.assertIn('::warning::fork-ci pressure: start', text) + self.assertIn('::warning::fork-ci-pressure: start', text) r = subprocess.run([sys.executable, '-I', SCRIPT, 'end'], capture_output=True, text=True, env={'PATH': os.environ.get('PATH', ''), 'RUNNER_TEMP': self.temp, 'JOB_STATUS': 'failure'}) self.assertEqual(r.returncode, 0, r.stderr) - self.assertIn('fork-ci pressure: job failure', r.stdout) + self.assertIn('fork-ci-pressure: job failure', r.stdout) class Steps(unittest.TestCase): diff --git a/.github/scripts/fork-ci-rerun.py b/.github/scripts/fork-ci-rerun.py index 4454941e5c96c..826496c26e7b1 100644 --- a/.github/scripts/fork-ci-rerun.py +++ b/.github/scripts/fork-ci-rerun.py @@ -33,7 +33,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) CONFIG = os.path.join(os.path.dirname(HERE), 'fork-ci-rerun.json') API = os.environ.get('GITHUB_API_URL') or 'https://api.github.com' -NOTE_TITLE = 'fork-ci pressure' # fork-ci-pressure.py's +NOTE_TITLE = 'fork-ci-pressure' # fork-ci-pressure.py's NOTE_HEAD = 'fork-ci-pressure v1 ' RED = ('failure', 'cancelled', 'timed_out') CHECK_RUN = re.compile(r'/check-runs/([0-9]{1,20})$') From 00274ddbb829cfb64229d6a0061215f1b210212e Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:02:52 +0000 Subject: [PATCH 3/4] fork CI re-run: signs only from GitHub's own words and the job's record, one tracking issue (#130) Review fixes for the one automatic re-run: - Text is never a sign: a message counts only in the runner's form (failure, no title, .github), as a whole GitHub message (fullmatch), and only when the job's steps agree (lost: no failed step, no time limit, its runner, the end step never ran; limit: the job or step really ran that long, no other step failed). A failing test that prints GitHub's words, or a forged note, is left red. - Memory full needs the job's own note (its end step completed, one note) to show a limited box stalled AND events in it, on a runner that is not GitHub-hosted; a machine-wide OOM kill or one event alone is no sign. A person's cancel never qualifies. - Every re-run (and fleet-mode report, once) is first a comment on one tracking issue, written before the request; the 24 h count reads it. - Every page of a listing is read (deduped, a cap that errors); trust is the fleet's own run_refused; a dispatch has a concurrency group of its own; the probe steps get a 1-minute timeout. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-rerun.json | 64 +- .github/fork-ci-rerun.md | 141 ++-- .github/scripts/fork-ci-pressure.test.py | 7 + .github/scripts/fork-ci-rerun.py | 481 +++++++++---- .github/scripts/fork-ci-rerun.test.py | 859 +++++++++++++++++------ .github/workflows/fork-ci-rerun.yml | 43 +- .github/workflows/fork-tests.yml | 14 + 7 files changed, 1175 insertions(+), 434 deletions(-) diff --git a/.github/fork-ci-rerun.json b/.github/fork-ci-rerun.json index 9707660272815..13c9550136003 100644 --- a/.github/fork-ci-rerun.json +++ b/.github/fork-ci-rerun.json @@ -3,33 +3,59 @@ "Fork only: when a failed Fork tests run gets its one automatic re-run. Read by .github/scripts/fork-ci-rerun.py", "(.github/workflows/fork-ci-rerun.yml every 10 minutes); the script has no fact about this repository, these are", "all of them. Runbook: .github/fork-ci-rerun.md.", - "signs: regular expressions matched against a failed job's annotations (GitHub's own messages).", - " lost: the runner went away; enough on its own.", - " limit: the job or a step ran past its time limit; only with the job's pressure note showing `stalled`.", - "stalled: the job's memory pressure note (.github/scripts/fork-ci-pressure.py) shows the machine or the job's box", - " stalled: some-or-all-tasks-waiting (full) share of the job's time, or the 5-minute full average at its end.", - "memory_full: any failure, when the note shows the box full: OOM kills, memory.max hits, or memory.high", - " throttling with a full stall share at least high_events_with_full_share_percent.", + "trust: the fleet config beside this one; its repository, trusted workflows, actors and branches, and its rule", + " (fork-fleet.py run_refused), decide which runs are trusted. workflow must be one of its trusted workflows.", + "list_hours: how far back runs are listed (by creation; every page is read); catch_up_minutes: of those, the", + " runs that ended (updated) this recently. window_hours: the re-run count read from the tracking issue.", + "hosted_runner_groups: GitHub-hosted runners' group; an OOM kill there is the job's own (no box of ours).", + "probe_end_step: the step fork-ci-pressure.py writes the note in (fork-tests.yml); only a job that completed it", + " has a note of its own, and a job that never reached it lost its runner or was stopped.", + "limit_slack_seconds: a time-limit message counts only when the job (or the step) ran its minutes less this.", + "signs: GitHub's own runner messages, each pattern a whole message (re.fullmatch), only on an annotation in the", + " runner's form (failure, no title, .github), and only together with the job's step record (the script's rules).", + " lost: the runner went away. limit: the job or a step ran past its time limit; only with `stalled`.", + " cancelled_by: a person cancelled it; the run stays red.", + "stalled: the job's memory pressure note shows the machine or a box stalled: some-or-all-tasks-waiting (full) share", + " of the job's time, or the 5-minute full average at its end.", + "memory_full: on a runner that is not GitHub-hosted, a box with a memory limit stalled (full) at least", + " box_full_share_percent of the job, AND in that box OOM kills, memory.max hits or memory.high throttling.", "skip_when: with this environment variable (the workflow sets it from a repository variable) at one of these", - " values, a run that qualifies is only reported, not re-run (fleet mode: a re-run of failed jobs would wait for", - " fleet runners that refuse it, .github/fork-fleet.md)." + " values, a run that qualifies is only reported, once, not re-run (fleet mode: a re-run of failed jobs would wait", + " for fleet runners that refuse it, .github/fork-fleet.md).", + "notice: the one tracking issue every re-run (and report) is a comment on; opened by `author` when none is open.", + " labels: given when it is opened (each must exist in the repository)." ], - "repository": "scuttletoby/t3code", + "trust": "fork-fleet.json", "workflow": "fork-tests.yml", - "actors": ["scuttletoby"], - "events": ["push", "pull_request"], - "push_branches": ["main"], "followers": ["fork-ci-result"], - "window_hours": 24, + "list_hours": 6, "catch_up_minutes": 60, + "window_hours": 24, "skip_when": { "variable": "CI_RUNNERS", "is": ["fleet"] }, + "hosted_runner_groups": ["GitHub Actions"], + "probe_end_step": "Note memory pressure at the end", + "limit_slack_seconds": 60, "signs": { - "lost": ["lost communication with the server", "The runner has received a shutdown signal"], + "lost": [ + "The (?:self-hosted|hosted) runner: (?P.+?) lost communication with the server\\..*", + "The runner has received a shutdown signal\\..*" + ], "limit": [ - "has exceeded the maximum execution time of", - "The action '[^']*' has timed out after" - ] + "The job running on runner (?P.+) has exceeded the maximum execution time of (?P[0-9]+) minutes\\.", + "The action '(?P[^']+)' has timed out after (?P[0-9]+) minutes\\." + ], + "cancelled_by": ["The run was canceled by @[^ ]+\\."] }, "stalled": { "full_share_percent": 10, "full_avg300_percent": 10 }, - "memory_full": { "oom_kills": 1, "max_events": 1, "high_events_with_full_share_percent": 10 } + "memory_full": { + "box_full_share_percent": 10, + "oom_kills": 1, + "max_events": 1, + "high_events": 1 + }, + "notice": { + "title": "Fork CI: automatic re-runs (infrastructure)", + "labels": [], + "author": "github-actions[bot]" + } } diff --git a/.github/fork-ci-rerun.md b/.github/fork-ci-rerun.md index 245bbc87c10b6..f1210197ecf6e 100644 --- a/.github/fork-ci-rerun.md +++ b/.github/fork-ci-rerun.md @@ -1,17 +1,19 @@ # Fork CI re-run: one automatic re-run for infrastructure failures only A failed `Fork tests` run gets **one** automatic "re-run failed jobs" (GitHub's -own), only when every failed job shows a known infrastructure sign. Never for a -test failure, never twice: a repeat red is left red, and it goes to an issue -and its fix first. - -| Part | What it does | -| --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `.github/scripts/fork-ci-pressure.py` | Two steps in each app job of `fork-tests.yml` (the routable jobs of `fork-fleet.json`): `start` after the checkout, `end` last (`if: always()`). Facts only, never fails the job. | -| `.github/scripts/fork-ci-rerun.py` | The decision, with every rule in one place, and the re-run. | -| `.github/fork-ci-rerun.json` | Every fact and number the decision uses: repository, workflow, trusted actors and branches, followers, sign patterns, thresholds, the fleet-mode switch. | -| `.github/workflows/fork-ci-rerun.yml` | Runs the decision every 10 minutes (minutes 5, 15, ...), GitHub-hosted, `actions: write`, `checks: read`. `workflow_dispatch` with `run_id` checks one run. | -| `fork-ci-rerun.test.py`, `fork-ci-pressure.test.py` | Each sign, each refusal, the state cases below, the wiring. | +own), only when every failed job shows a known infrastructure sign that +GitHub's own runner message and the job's own step record agree on. Never for +a test failure, never for a person's cancel, never twice: a repeat red is left +red, and it goes to an issue and its fix first. Every re-run is also a comment +on one tracking issue, so a machine that keeps stalling is never hidden. + +| Part | What it does | +| --------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `.github/scripts/fork-ci-pressure.py` | Two steps in each app job of `fork-tests.yml` (the routable jobs of `fork-fleet.json`): `start` after the checkout, `end` last (`if: always()`), each with a 1-minute timeout. Facts only: it never fails (it catches every error), and a hung one never holds up the job. | +| `.github/scripts/fork-ci-rerun.py` | The decision, with every rule in one place, the tracking issue, and the re-run. Uses `fork-fleet.py`'s trust rule. | +| `.github/fork-ci-rerun.json` | Every fact and number the decision uses: the trust config (`fork-fleet.json`), workflow, followers, windows, the hosted runner group, the probe's end step, sign patterns, thresholds, the fleet-mode switch, the tracking issue. | +| `.github/workflows/fork-ci-rerun.yml` | Runs the decision every 10 minutes (minutes 5, 15, ...), GitHub-hosted, `actions: write`, `checks: read`, `issues: write`. `workflow_dispatch` with `run_id` checks one run, in a concurrency group of its own. | +| `fork-ci-rerun.test.py`, `fork-ci-pressure.test.py` | Each sign, each refusal (each rule line is pinned: removing it turns a test red), the state cases below, the paging, the wiring. | ## The numbers in the job log @@ -39,73 +41,106 @@ start sample, a reboot in between, a file the runner cannot see. ## The rules (fork-ci-rerun.py) -1. Only a trusted run: this repository, `fork-tests.yml`, started and - triggered by a listed actor, a push to a listed branch or a pull request - from a branch of this repository. +1. Only a trusted run, by the fleet's own rule (`fork-fleet.py` `run_refused` + with `fork-fleet.json`): this repository, a trusted workflow, started and + triggered by a trusted actor, a push to a trusted branch not made by the + upstream sync, or a pull request a trusted actor opened from a branch of + this repository. Never looser than what may run on the fleet. 2. Never twice: if any attempt of the run was triggered by someone else (the re-run is triggered by `github-actions`), the run is left red. 3. Each failed job (failure, cancelled, timed out) except the followers - (`fork-ci-result`, which needs every job) must show a sign: + (`fork-ci-result`, which needs every job) must show a sign. **Text is never + enough**: a step can print any annotation (`::error::` with GitHub's own + words, `::notice title=fork-ci-pressure::`), with exactly the fields the + runner's own have. So a message counts only in the runner's form (failure + level, no title, on `.github`), only as a whole message (`re.fullmatch` of a + `signs` pattern), and only when the job's own record agrees: - **lost**: GitHub says the runner lost communication with the server, or - received a shutdown signal. Enough on its own (no note needed: the job - never reached its last step). - - **memory full**: the note shows an OOM kill on the machine or in the box, - the box hitting `memory.max`, or `memory.high` throttling while stalled - (full) at least `high_events_with_full_share_percent` of the job. + received a shutdown signal; the job ended `failure` on that runner, none + of its steps failed, no time limit was hit, and the probe's end step never + completed (the runner died before it). No note needed. + - **memory full**: the job ended `failure` on a runner that is not + GitHub-hosted (`hosted_runner_groups`: an OOM kill there is the job's + own), and its own note shows a box with a memory limit stalled (full) at + least `box_full_share_percent` of the job AND, in that box, OOM kills, + `memory.max` hits or `memory.high` throttling. One event without the + stall, or a machine-wide OOM kill, is no sign. - **limit while stalled**: GitHub says the job or a step ran past its time - limit, and the note shows the machine or the box stalled (full) at least - `full_share_percent` of the job, or a 5-minute full average at least - `full_avg300_percent` at the end. -4. One failed job without a sign leaves the run red: a test failure, a cancel - by a person or a newer push, a job with no note or a garbled one, more than - one note. Fail closed. -5. The branch must still be at the run's commit (a newer push tests it anyway). -6. With `CI_RUNNERS=fleet` (`skip_when`), a run that qualifies is reported with - an error annotation and not re-run: a re-run of failed jobs keeps its fleet - labels and would wait for fleet runners that refuse the bot's attempt - (`fork-fleet.md`). Re-run it by hand. + limit; the job (or that step) really ran that long (less + `limit_slack_seconds`); no other step failed; and the note shows the + machine or a box stalled (full) at least `full_share_percent` of the job, + or a 5-minute full average at least `full_avg300_percent` at the end. +4. The note counts only when the probe's end step completed (so the job wrote + its own) and it is the job's only one, a notice on `.github` with the + `fork-ci-pressure v1 ` header and the right shape. +5. One failed job without a sign leaves the run red: a test failure, a job + with no note, a garbled one, two notes. GitHub's "The run was canceled by + @..." on a failed job leaves it red too: a person's cancel stands, whatever + the memory was doing. Fail closed. +6. The branch must still be at the run's commit (a newer push tests it anyway). +7. With `CI_RUNNERS=fleet` (`skip_when`), a run that qualifies is reported, once + (one comment, one error annotation), and not re-run: a re-run of failed jobs + keeps its fleet labels and would wait for fleet runners that refuse the + bot's attempt (`fork-fleet.md`). Re-run it by hand. ## Visible and counted -- Each re-run is a **warning annotation** on the `Fork CI re-run` run, with - the job names, the signs, GitHub's message and the numbers. -- Each pass writes a **job summary**: a table of every red run of the last hour - and what was done (re-run, left red and why), and the count of runs whose - latest attempt was an automatic re-run in the last 24 hours (this one's and - the fleet sweep's). A stalled machine keeps showing up there and in the - failed job's own log line; the re-run does not hide it. +- **The tracking issue** (`notice` in the config): the first re-run opens one + issue, by `github-actions`, with that title; every later re-run (and every + fleet-mode report) is one comment on it: the run's link, event, branch and + commit, each job's sign, GitHub's message and the pressure numbers. Never one + issue per re-run. Only an open issue by `github-actions` with exactly that + title is used; closing it is fine (the next re-run opens a new one). A + comment carries a marker line (``); the comment is written **before** the request. +- Each re-run is also a **warning annotation** on the `Fork CI re-run` run. +- Each pass writes a **job summary**: a table of every red run it looked at and + what was done, and the count of automatic re-runs of the last 24 hours, read + from the issue's markers (two to a few API calls a pass, however many runs). ## State -Nothing is kept between passes: each one reads GitHub. - -| Case | What happens | -| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| A pass runs again on a run it re-ran | The run is queued or running (skipped), or ended again with an attempt by `github-actions`: left red. | -| Two passes (or this and the fleet sweep) race | The run is read again just before the request. If GitHub refuses the second request and the run has a new attempt, that is fine. | -| A pass stops before its request reached GitHub | The run is unchanged; the next pass (within `catch_up_minutes`) sends it once. | -| A pass stops after GitHub took the request | The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | -| An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | -| A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). | -| GitHub errors on one run | A warning; the other runs are still handled, and the pass ends red so the error is seen. The next pass tries again. | +Nothing is kept between passes: each one reads GitHub (the runs and the issue). + +| Case | What happens | +| ------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A pass runs again on a run it re-ran | The run is queued or running (skipped), or ended again with an attempt by `github-actions`: left red. No second comment. | +| Two passes (or this and the fleet sweep) race | The run is read again just before the request. If GitHub refuses the second request and the run has a new attempt, that is fine. | +| Two passes open the issue at once | Each reads the open issues after opening its own; the lowest number wins and the other closes its own as a duplicate. | +| A pass stops before its request reached GitHub | The comment is there, the run unchanged; the next pass (within `catch_up_minutes`) sends the request once and adds no second comment. | +| A pass stops after GitHub took the request (reply lost) | The comment, written first, says why. The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | +| The comment cannot be written | No re-run (it would be hidden): a warning, the pass ends red, the next pass tries both. | +| A dispatch with `run_id` while a scheduled pass waits | Its own concurrency group: it is never replaced by a scheduled pass. | +| An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | +| A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). Runs are listed by creation over `list_hours`, every page read; a listing past 10 pages ends the pass red, nothing re-run. | +| GitHub errors on one run | A warning; the other runs are still handled, and the pass ends red so the error is seen. The next pass tries again. | +| No tracking issue, or it was closed | The count reads 0 from a new window; the next re-run opens a new one. | ## Overlap with the fleet sweep `fork-fleet.yml`'s sweep also re-runs (the whole run, GitHub-hosted) a run whose fleet runner was lost, in `auto` mode. Both stay at one re-run per run: each leaves alone a run with an attempt by `github-actions`, and GitHub refuses -a re-run while one is going. They are offset by 5 minutes. +a re-run while one is going. They are offset by 5 minutes. The fleet sweep's +own re-runs are not comments on the tracking issue (its own job summary shows +them); that is `fork-fleet.py`'s to change. ## Limits - No live proof before merge: scheduled workflows run from the default branch. - After merge, dispatch `Fork CI re-run` once and read its summary. + After merge, dispatch `Fork CI re-run` once and read its summary. Then check, + on a real red run, what the jobs API says (a) for the end step after a job + time limit (does `if: always()` still run it? if not, `limit` never + qualifies, fail closed) and (b) for a `continue-on-error` step that failed; + and on a real lost runner, its steps' state. +- A note is the job's own words: a trusted run's code could still print one + and keep the real end step from writing its own (two notes are refused, and + a note without its end step's completion is not read). Only trusted runs are + read, and only once. - An OOM kill by a user-space killer (earlyoom, systemd-oomd) is not in `memory.events` or `oom_kill`: it shows only as a stall, if at all. - GitHub keeps at most 10 notices per step and 50 annotations per job: on a job with very many annotations the note can be dropped. The job then reads as "no note" and stays red (fail closed). -- A trusted run's own code could print a forged note. Only trusted runs are - read, and two notes in one job are refused. - A job a step's own time limit failed counts as `limit` only with a stall; a slow test on a calm machine stays red. diff --git a/.github/scripts/fork-ci-pressure.test.py b/.github/scripts/fork-ci-pressure.test.py index 1d0ea7cccc808..b7648e16d802d 100644 --- a/.github/scripts/fork-ci-pressure.test.py +++ b/.github/scripts/fork-ci-pressure.test.py @@ -210,6 +210,13 @@ def test_every_app_job_has_the_start_and_end_steps(self): r"\.py'\) != '' \}\}", end['if']), end['if']) self.assertEqual(end['env'], {'JOB_STATUS': '${{ job.status }}'}) self.assertEqual(sum('fork-ci-pressure' in str(s.get('run')) for s in steps), 2) + for probe in (steps[1], end): + # A hung probe never holds the job up to its own limit. No continue-on-error: nothing in + # fork-tests.yml may fail quietly (fork-contracts.test.py); the probe itself never fails. + self.assertIn(probe.get('timeout-minutes'), (1, 2), probe['name']) + self.assertNotIn('continue-on-error', probe) + with open(os.path.join(ROOT, '.github/fork-ci-rerun.json'), encoding='utf-8') as f: + self.assertEqual(end['name'], json.load(f)['probe_end_step']) # the re-run reads its record def test_the_probe_is_not_fork_owned_app_jobs_read_it(self): with open(os.path.join(ROOT, '.github/fork-owned-paths.txt'), encoding='utf-8') as f: diff --git a/.github/scripts/fork-ci-rerun.py b/.github/scripts/fork-ci-rerun.py index 826496c26e7b1..cb78aac91c073 100644 --- a/.github/scripts/fork-ci-rerun.py +++ b/.github/scripts/fork-ci-rerun.py @@ -2,34 +2,49 @@ """Fork only: one automatic re-run of a failed Fork tests run, only for a known infrastructure sign. python3 -I fork-ci-rerun.py sweep [--run-id N] .github/workflows/fork-ci-rerun.yml (every 10 minutes, GitHub-hosted, - actions: write, checks: read): each failed trusted run of the last - catch_up_minutes (or only run N) whose every failed job shows a sign - gets GitHub's own "re-run failed jobs", once. + actions: write, checks: read, issues: write): each failed trusted run + that ended in the last catch_up_minutes (or only run N) whose every + failed job shows a sign gets GitHub's own "re-run failed jobs", once. python3 -I fork-ci-rerun.py check checks CONFIG (fork-ci-rerun.test.py runs it). THE rules, in one place (the facts, patterns and numbers are CONFIG's, .github/fork-ci-rerun.json; none is here): -- Only a trusted run: of CONFIG's repository and workflow, started and triggered by one of its actors, a push to one - of its branches or a pull request whose head is this repository. Anything else is left alone. +- Only a trusted run, by the fleet's own trust rule (fork-fleet.py run_refused, with the config CONFIG `trust` + names): never looser than the rule that decides what may run on the fleet. - Never twice: a run any of whose attempts was triggered by someone else (GitHub's re-run from this script, or the fleet sweep's, is triggered by github-actions) is left red. A repeat red goes to an issue and its fix first. - Each failed job (failure, cancelled, timed out) but CONFIG's followers (jobs that need all the others and fail - only because one of them did) must show a sign, from its annotations (GitHub's own messages) and its pressure - note (fork-ci-pressure.py): - lost the runner lost communication or was shut down (CONFIG signs.lost): enough on its own; - memory full any failure, when the note shows the job's box or machine full (CONFIG memory_full); - limit a job or step past its time limit (CONFIG signs.limit), only when the note shows the machine or - box stalled (CONFIG stalled). - One failed job without a sign (a test failure, a person's cancel, no note, a garbled note) leaves the run red: - fail closed. A run whose branch has moved on, or is gone, is not re-run. -- With CONFIG skip_when (fleet mode), a run that qualifies is only reported, with an error annotation. -Visible and counted: each re-run is a warning annotation with its signs and numbers, a line in this job's summary, -and every pass counts the runs a bot re-ran in the window. Nothing is kept between passes: each reads GitHub, so a -pass that stopped half-way is finished by the next, and one that runs again finds the bot's attempt and does nothing. + only because one of them did) must show a sign. Text alone is never a sign: a step can print any annotation + (::error::, ::notice title=...::), so a message counts only in the form the runner itself writes (failure level, + no title, on .github, the whole message one of CONFIG's patterns), and only when the job's own step record + agrees: + lost GitHub says the runner lost communication or was shut down (signs.lost), the job ended failure + on that runner, none of its steps failed, no time limit was hit, and the probe's end step + (probe_end_step) never completed; + memory full the job ended failure on a runner that is not GitHub-hosted (hosted_runner_groups: a kill there is + the job's own), and its own pressure note shows a box with a memory limit stalled (full) at least + memory_full.box_full_share_percent of the job AND OOM kills, memory.max hits or memory.high + throttling in it. A machine-wide OOM kill, or one event without the stall, is no sign; + limit GitHub says the job or a step ran past its time limit (signs.limit, with the minutes), the job (or + that step) really ran that long, no other step failed, and the note shows the machine or the box + stalled (CONFIG stalled). + The note counts only when the probe's end step completed (so the job wrote its own) and is the job's only one. + One failed job without a sign (a test failure, no note, a garbled note, two notes) leaves the run red: fail + closed. GitHub's "canceled by" (signs.cancelled_by) on any failed job leaves it red: a person's cancel stands. + A run whose branch has moved on, or is gone, is not re-run. +- With CONFIG skip_when (fleet mode), a run that qualifies is only reported, once. +Visible and durable: every re-run (and every fleet-mode report) is first a comment on one tracking issue (CONFIG +notice: opened once, by github-actions, then a comment per re-run with the run, the signs and the numbers), then a +warning annotation and a row in this job's summary, which also counts the issue's re-runs of the last window_hours. +The comment is written before the request, so a reply lost after GitHub took it still leaves the reason. Nothing +else is kept between passes: each reads GitHub, so a pass that stopped half-way is finished by the next, and one that +runs again finds the bot's attempt (and the issue's marker) and does nothing. Runbook: .github/fork-ci-rerun.md. Standard library only. """ -import argparse, calendar, http.client, json, os, re, sys, time +import argparse, calendar, http.client, importlib.util, json, os, re, sys, time import urllib.error, urllib.parse, urllib.request +sys.dont_write_bytecode = True # loading fork-fleet.py must not leave __pycache__ in the tree + HERE = os.path.dirname(os.path.abspath(__file__)) CONFIG = os.path.join(os.path.dirname(HERE), 'fork-ci-rerun.json') API = os.environ.get('GITHUB_API_URL') or 'https://api.github.com' @@ -37,6 +52,14 @@ NOTE_HEAD = 'fork-ci-pressure v1 ' RED = ('failure', 'cancelled', 'timed_out') CHECK_RUN = re.compile(r'/check-runs/([0-9]{1,20})$') +MARK = re.compile(r'') +# Each sign's named groups: `minutes` is how long GitHub says the limit was; `step` (when there) names the step. +GROUPS = {'lost': set(), 'limit': {'minutes'}, 'cancelled_by': set()} +LONGEST = 60000 # characters of a comment (GitHub takes 65536) + +_spec = importlib.util.spec_from_file_location('fork_fleet', os.path.join(HERE, 'fork-fleet.py')) +FLEET = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(FLEET) def say(msg, out=None): @@ -51,45 +74,53 @@ def load(path=CONFIG): problems = check_config(cfg) if problems: raise ValueError(f'{path}: ' + '; '.join(problems)) - return cfg + fleet = FLEET.load(os.path.join(os.path.dirname(os.path.abspath(path)), cfg['trust'])) + workflows = [os.path.basename(w) for w in fleet['trusted']['workflows']] + if cfg['workflow'] not in workflows: + raise ValueError(f'{path}: workflow {cfg["workflow"]!r} is not one the trust config trusts ({workflows})') + return dict(cfg, fleet=fleet) def number(x): return isinstance(x, (int, float)) and not isinstance(x, bool) -def names(x): - return isinstance(x, list) and bool(x) and all(isinstance(s, str) and s for s in x) +def names(x, empty=False): + return isinstance(x, list) and (empty or bool(x)) and all(isinstance(s, str) and s for s in x) def check_config(cfg): out = [] if not isinstance(cfg, dict): return ['not a JSON object'] - known = {'_doc', 'repository', 'workflow', 'actors', 'events', 'push_branches', 'followers', 'window_hours', - 'catch_up_minutes', 'skip_when', 'signs', 'stalled', 'memory_full'} + known = {'_doc', 'trust', 'workflow', 'followers', 'list_hours', 'catch_up_minutes', 'window_hours', 'skip_when', + 'hosted_runner_groups', 'probe_end_step', 'limit_slack_seconds', 'signs', 'stalled', 'memory_full', + 'notice'} out += [f'unknown key {k!r}' for k in sorted(set(cfg) - known)] - if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', str(cfg.get('repository', ''))): - out.append('repository must be owner/name') + out += [f'missing key {k!r}' for k in sorted(known - {'_doc'} - set(cfg))] + if not re.fullmatch(r'[A-Za-z0-9_.-]+\.json', str(cfg.get('trust', ''))): + out.append('trust must be the file name of the fleet config beside this one') if not re.fullmatch(r'[A-Za-z0-9_.-]+\.ya?ml', str(cfg.get('workflow', ''))): out.append('workflow must be a workflow file name') - for key in ('actors', 'events', 'push_branches'): - if not names(cfg.get(key)): - out.append(f'{key} must be a list of names') - if not set(cfg.get('events') or []) <= {'push', 'pull_request'}: - out.append('events: only push and pull_request') - if not isinstance(cfg.get('followers'), list) or not all(isinstance(s, str) for s in cfg.get('followers', [])): + if not names(cfg.get('followers'), empty=True): out.append('followers must be a list of job names') - for key in ('window_hours', 'catch_up_minutes'): + for key in ('list_hours', 'catch_up_minutes', 'window_hours', 'limit_slack_seconds'): if not number(cfg.get(key)) or cfg[key] <= 0: out.append(f'{key} must be a positive number') + if number(cfg.get('list_hours')) and number(cfg.get('catch_up_minutes')) \ + and cfg['list_hours'] * 60 <= cfg['catch_up_minutes']: + out.append('list_hours must reach further back than catch_up_minutes') skip = cfg.get('skip_when') if not (isinstance(skip, dict) and re.fullmatch(r'[A-Z][A-Z0-9_]*', str(skip.get('variable', ''))) and names(skip.get('is'))): out.append('skip_when must be {"variable": NAME, "is": [values]}') + if not names(cfg.get('hosted_runner_groups')): + out.append('hosted_runner_groups must be a list of runner group names') + if not (isinstance(cfg.get('probe_end_step'), str) and cfg['probe_end_step']): + out.append('probe_end_step must be the name of the probe\'s end step') signs = cfg.get('signs') - if not isinstance(signs, dict) or set(signs) != {'lost', 'limit'}: - out.append('signs must have exactly lost and limit') + if not isinstance(signs, dict) or set(signs) != set(GROUPS): + out.append(f'signs must have exactly {", ".join(sorted(GROUPS))}') else: for kind, patterns in signs.items(): if not names(patterns): @@ -97,15 +128,25 @@ def check_config(cfg): continue for p in patterns: try: - re.compile(p) + groups = set(re.compile(p).groupindex) except re.error as e: out.append(f'signs.{kind}: {p!r}: {e}') + continue + if not GROUPS[kind] <= groups: + out.append(f'signs.{kind}: {p!r} lacks the group(s) {sorted(GROUPS[kind] - groups)}') + if not p.startswith('The '): + out.append(f'signs.{kind}: {p!r}: a whole GitHub message, from its first word') for key, fields in (('stalled', ('full_share_percent', 'full_avg300_percent')), - ('memory_full', ('oom_kills', 'max_events', 'high_events_with_full_share_percent'))): + ('memory_full', ('box_full_share_percent', 'oom_kills', 'max_events', 'high_events'))): got = cfg.get(key) if not isinstance(got, dict) or set(got) != set(fields) or not all(number(got[f]) and got[f] > 0 for f in fields): out.append(f'{key} must have exactly {", ".join(fields)}, each a positive number') + notice = cfg.get('notice') + if not (isinstance(notice, dict) and set(notice) == {'title', 'labels', 'author'} + and isinstance(notice.get('title'), str) and 0 < len(notice['title']) <= 200 + and names(notice.get('labels'), empty=True) and re.fullmatch(r'[a-z0-9-]+\[bot\]', str(notice.get('author')))): + out.append('notice must be {"title": TEXT, "labels": [names], "author": "[bot]"}') return out @@ -127,14 +168,24 @@ def call(self, path, method='GET', data=None): got = r.read() return json.loads(got) if got else {} - def pages(self, path, key, most=10): - out = [] + def pages(self, path, key=None, most=10): + """Every item of a listing (got[key], or the reply itself), page after page until a short one; each id once + (an item that moved down a page while new ones arrived is listed twice). More than `most` full pages is a + ValueError: a listing cut short would be read as complete.""" + out, seen = [], set() for page in range(1, most + 1): - got = self.call(f'{path}{"&" if "?" in path else "?"}per_page=100&page={page}')[key] - out += got - if len(got) < 100: + got = self.call(f'{path}{"&" if "?" in path else "?"}per_page=100&page={page}') + items = got[key] if key else got + if not isinstance(items, list): + raise ValueError(f'{path}: page {page} is not a list') + for item in items: + ident = item.get('id') if isinstance(item, dict) else None + if ident is None or ident not in seen: + seen.add(ident) + out.append(item) + if len(items) < 100: return out - return out + raise ValueError(f'{path}: more than {most} pages of 100; not read as complete') # A truncated reply is http.client.IncompleteRead (an HTTPException), a garbled one a ValueError. @@ -149,46 +200,70 @@ def stamp(text): return None +def iso(t): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t)) + + def login(x): return (x or {}).get('login') +def took(x): + """Seconds from a job's or step's started_at to its completed_at, or None.""" + a, b = stamp(x.get('started_at')), stamp(x.get('completed_at')) + return b - a if a is not None and b is not None and b >= a else None + + # ---------------------------------------------------------------- the rules -def untrusted(cfg, run): - """Why GitHub's record of a run is not one this script may re-run, or None.""" - repo = cfg['repository'] - if (run.get('repository') or {}).get('full_name') != repo: - return f'of another repository ({(run.get("repository") or {}).get("full_name")!r})' - if os.path.basename((run.get('path') or '').split('@')[0]) != cfg['workflow']: - return f'another workflow ({run.get("path")!r})' - if run.get('event') not in cfg['events']: - return f'event {run.get("event")!r}' - if login(run.get('actor')) not in cfg['actors']: - return f'started by {login(run.get("actor"))!r}' - if run.get('event') == 'push' and run.get('head_branch') not in cfg['push_branches']: - return f'a push to {run.get("head_branch")!r}' - if run.get('event') == 'pull_request' and (run.get('head_repository') or {}).get('full_name') != repo: - return 'a pull request from another repository' +def from_runner(a): + """An annotation in the form the runner writes its own job messages: failure level, no title, on .github. A step + can print one that looks the same (::error::), so a sign read from one needs the job's step record too.""" + return a.get('annotation_level') == 'failure' and not a.get('title') and a.get('path') == '.github' + + +def github_says(cfg, kind, annotations): + """(message, match) of the first runner-form annotation whose whole message is one of signs[kind], or None.""" + for a in annotations: + if not from_runner(a): + continue + msg = str(a.get('message') or '').strip() + for p in cfg['signs'][kind]: + m = re.fullmatch(p, msg, re.S) + if m: + return msg, m return None -def note_of(annotations): - """The job's pressure note (fork-ci-pressure.py) as a dict, or (None, why).""" +def end_step_ran(cfg, job): + """Did the probe's end step complete? Then the runner lived to the job's end, and the note (if any) is its.""" + return any(s.get('name') == cfg['probe_end_step'] and s.get('status') == 'completed' + and s.get('conclusion') == 'success' for s in job.get('steps') or []) + + +def failed_steps(job): + return [s.get('name') for s in job.get('steps') or [] if s.get('conclusion') == 'failure'] + + +def note_of(cfg, job, annotations): + """The job's own pressure note (fork-ci-pressure.py) as a dict, or (None, why).""" + if not end_step_ran(cfg, job): + return None, f'its "{cfg["probe_end_step"]}" step did not complete, so it has no note of its own' notes = [a for a in annotations if a.get('title') == NOTE_TITLE] if not notes: - return None, 'no pressure note (the job had no probe, or the note was dropped)' + return None, 'no pressure note (the note was dropped)' if len(notes) > 1: - return None, f'{len(notes)} pressure notes, not one' - msg = notes[0].get('message') or '' - if not msg.startswith(NOTE_HEAD): + return None, f'{len(notes)} pressure notes, not one (a step printed one too)' + a = notes[0] + msg = str(a.get('message') or '') + if a.get('annotation_level') != 'notice' or a.get('path') != '.github' or not msg.startswith(NOTE_HEAD): return None, 'a garbled pressure note' try: note = json.loads(msg[len(NOTE_HEAD):]) except ValueError: return None, 'a garbled pressure note' if not (isinstance(note, dict) and isinstance(note.get('host'), dict) and isinstance(note.get('cgroups'), list) - and all(isinstance(c, dict) for c in note['cgroups'])): + and all(isinstance(c, dict) for c in note['cgroups']) and isinstance(note.get('status'), str)): return None, 'a garbled pressure note' return note, None @@ -210,23 +285,64 @@ def stalled(cfg, note): return None -def memory_full(cfg, note): - """Evidence that the job's box (or the machine) was full, or None.""" - mf, host = cfg['memory_full'], note['host'] - if at_least(host.get('oom_kill'), mf['oom_kills']): - return f'the kernel OOM-killed {host["oom_kill"]} process(es) on the machine during the job' +def hosted(cfg, job): + group = job.get('runner_group_name') + return not (isinstance(group, str) and group) or group in cfg['hosted_runner_groups'] + + +def memory_full(cfg, job, note): + """Evidence that the job's own box was full and stalled, or None. Only a failure on a runner that is not + GitHub-hosted, only a box with a memory limit, and only events together with that box's stall.""" + if job.get('conclusion') != 'failure' or note.get('status') != 'failure' or hosted(cfg, job): + return None + mf = cfg['memory_full'] for c in note['cgroups']: - where = f'box {c.get("path")} (max {c.get("max_mb")} MB, high {c.get("high_mb")} MB)' + if not (number(c.get('max_mb')) or number(c.get('high_mb'))): + continue # no limit: not a box + if not at_least(c.get('full_share'), mf['box_full_share_percent']): + continue + where = (f'box {c.get("path")} (max {c.get("max_mb")} MB, high {c.get("high_mb")} MB), stalled (full) ' + f'{c["full_share"]}% of the job') if at_least(c.get('oom_kill'), mf['oom_kills']): - return f'{where}: {c["oom_kill"]} OOM kill(s) during the job' + return f'{where}: {c["oom_kill"]} OOM kill(s) in it' if at_least(c.get('max'), mf['max_events']): - return f'{where}: hit its memory.max {c["max"]} time(s) during the job' - share = max([s for s in (c.get('full_share'), host.get('full_share')) if number(s)], default=None) - if at_least(c.get('high'), 1) and at_least(share, mf['high_events_with_full_share_percent']): - return f'{where}: throttled at memory.high {c["high"]} time(s), stalled (full) {share}% of the job' + return f'{where}: hit its memory.max {c["max"]} time(s)' + if at_least(c.get('high'), mf['high_events']): + return f'{where}: throttled at memory.high {c["high"]} time(s)' return None +def limit_hit(cfg, job, annotations): + """GitHub's time-limit message, when the job's record agrees: the job (or the step it names) ran at least that + long and no other step failed. (message, None), or (None, why) when there is no such message.""" + why = None + for a in annotations: + if not from_runner(a): + continue + msg = str(a.get('message') or '').strip() + for p in cfg['signs']['limit']: + m = re.fullmatch(p, msg, re.S) + if not m: + continue + bound = int(m['minutes']) * 60 - cfg['limit_slack_seconds'] + step = m.groupdict().get('step') + others = [s for s in failed_steps(job) if s != step] + if others: + why = f'GitHub\'s time-limit text, but step {others[0]!r} failed' + continue + if step is not None: + ran = [took(s) or 0 for s in job.get('steps') or [] if s.get('name') == step + and s.get('conclusion') in ('failure', 'cancelled')] + if not any(t >= bound for t in ran): + why = f'GitHub\'s time-limit text, but no step {step!r} ran {m["minutes"]} min and ended red' + continue + elif (took(job) or 0) < bound: + why = f'GitHub\'s time-limit text, but the job ran {int((took(job) or 0) // 60)} min, not {m["minutes"]}' + continue + return msg, None + return None, why + + def numbers(note): """The note's numbers, for the log.""" h = note['host'] @@ -239,41 +355,45 @@ def numbers(note): return '; '.join(out) -def matches(patterns, messages): - for m in messages: - for p in patterns: - if re.search(p, m): - return m - return None - - -def job_sign(cfg, annotations): +def job_sign(cfg, job, annotations): """(sign, evidence) when a failed job shows an infrastructure sign, else (None, why not).""" - messages = [str(a.get('message') or '') for a in annotations if a.get('title') != NOTE_TITLE] - hit = matches(cfg['signs']['lost'], messages) - if hit: - return 'lost', f'GitHub: "{hit.strip()[:200]}"' - note, why = note_of(annotations) - full = memory_full(cfg, note) if note else None + cancel = github_says(cfg, 'cancelled_by', annotations) + if cancel: + return None, f'GitHub: "{cancel[0][:200]}": a person\'s cancel stands' + note, why = note_of(cfg, job, annotations) + lost = github_says(cfg, 'lost', annotations) + if lost: + msg = f'GitHub: "{lost[0][:200]}"' + if job.get('conclusion') != 'failure': + return None, f'{msg}, but the job ended {job.get("conclusion")}, not failure' + if failed_steps(job): + return None, f'{msg}, but step {failed_steps(job)[0]!r} failed: the text is a step\'s output' + if github_says(cfg, 'limit', annotations): + return None, f'{msg}, but GitHub also says it ran past a time limit: a step that hung, not a lost runner' + runner = lost[1].groupdict().get('runner') + if runner is not None and job.get('runner_name') and runner != job['runner_name']: + return None, f'{msg}, but the job ran on {job["runner_name"]!r}: not its runner' + if end_step_ran(cfg, job): + return None, f'{msg}, but its "{cfg["probe_end_step"]}" step completed: the runner was not lost' + return 'lost', f'{msg}; no step failed and the job never reached its end step (no numbers: the runner died)' + full = memory_full(cfg, job, note) if note else None if full: return 'memory full', f'{full} ({numbers(note)})' - hit = matches(cfg['signs']['limit'], messages) + hit, not_hit = limit_hit(cfg, job, annotations) if hit and note: stall = stalled(cfg, note) if stall: - return 'limit while stalled', f'GitHub: "{hit.strip()[:200]}"; {stall} ({numbers(note)})' + return 'limit while stalled', f'GitHub: "{hit[:200]}"; {stall} ({numbers(note)})' return None, f'ran past its time limit, but the machine was not stalled ({numbers(note)})' if hit: return None, f'ran past its time limit, and {why}: cannot tell it was stalled' - return None, 'no infrastructure sign (a failure of its own, or a cancel)' + (f'; {why}' if not note else '') + return None, ('no infrastructure sign (a failure of its own, or a cancel)' + (f'; {not_hit}' if not_hit else '') + + (f'; {why}' if not note else '')) def annotations_of(gh, job): m = CHECK_RUN.search(job.get('check_run_url') or '') - got = gh.call(f'check-runs/{int(m[1]) if m else int(job["id"])}/annotations?per_page=100') - if not isinstance(got, list): - raise ValueError('annotations: not a list') - return got + return gh.pages(f'check-runs/{int(m[1]) if m else int(job["id"])}/annotations', most=3) def current(gh, run): @@ -292,16 +412,16 @@ def decide(cfg, gh, run, env): rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) if run.get('status') != 'completed' or run.get('conclusion') not in RED: return 'quiet', None - why = untrusted(cfg, run) + actors = cfg['fleet']['trusted']['actors'] + by = login(run.get('triggering_actor')) + if attempt > 1 and by not in actors: + return 'leave', f'attempt {attempt} was triggered by {by!r} (an automatic re-run): a repeat red stays red' + why = FLEET.run_refused(cfg['fleet'], gh, run) if why: return 'leave', f'not a trusted run: {why}' - by = login(run.get('triggering_actor')) - if by not in cfg['actors']: - return 'leave', (f'attempt {attempt} was triggered by {by!r} (an automatic re-run): a repeat red stays red' - if attempt > 1 else f'triggered by {by!r}: not a trusted run') for n in range(attempt - 1, 0, -1): by = login(gh.call(f'actions/runs/{rid}/attempts/{n}').get('triggering_actor')) - if by not in cfg['actors']: + if by not in actors: return 'leave', f'attempt {n} was triggered by {by!r} (an automatic re-run): never twice' jobs = gh.pages(f'actions/runs/{rid}/jobs?filter=latest', 'jobs') red = [j for j in jobs if j.get('conclusion') in RED and j.get('name') not in cfg['followers']] @@ -309,7 +429,7 @@ def decide(cfg, gh, run, env): return 'leave', 'no failed job of its own (only followers failed, or nothing did)' evidence = [] for j in red: - sign, why = job_sign(cfg, annotations_of(gh, j)) + sign, why = job_sign(cfg, j, annotations_of(gh, j)) if not sign: return 'leave', f'{j.get("name")} ended {j.get("conclusion")}: {why}' evidence.append(f'{j.get("name")} ({j.get("conclusion")}): {sign}: {why}') @@ -321,12 +441,103 @@ def decide(cfg, gh, run, env): return 'rerun', ' | '.join(evidence) -def rerun(gh, run): - """GitHub's own re-run of failed jobs (and the jobs after them), once: (True, how) when this pass asked for it.""" +# ---------------------------------------------------------------- the tracking issue + +class Notice: + """The one tracking issue (CONFIG notice): the durable, visible record of every automatic re-run outside this + sweep's own run. Only an open issue github-actions opened with exactly that title is ours, never one a person + opened; with none open, the first re-run opens it. Each re-run (and each fleet-mode report) is one comment with a + marker line (MARK: run, attempt, what), so a pass never writes the same one twice and the count reads them.""" + + def __init__(self, cfg, gh, now): + self.cfg, self.gh, self.now, self.hours = cfg['notice'], gh, now, cfg['window_hours'] + self.issue, self.comments = None, None + + def _ours(self): + n = self.cfg + listed = self.gh.pages(f'issues?state=open&creator={urllib.parse.quote(n["author"], safe="")}' + f'&sort=created&direction=asc') + return sorted(int(i['number']) for i in listed if isinstance(i, dict) and 'pull_request' not in i + and i.get('title') == n['title'] and login(i.get('user')) == n['author']) + + def load(self, since=None): + if self.comments is not None: + return + ours = self._ours() + self.issue = ours[0] if ours else None + start = iso(min(self.now() - self.hours * 3600, since or self.now())) + self.comments = [] if self.issue is None else [ + c for c in self.gh.pages(f'issues/{self.issue}/comments?since={start}') + if isinstance(c, dict) and login(c.get('user')) == self.cfg['author']] + + def marks(self): + out = {} + for c in self.comments or []: + for m in MARK.finditer(str(c.get('body') or '')): + out.setdefault((int(m[1]), int(m[2]), m[3]), c.get('created_at')) + return out + + def has(self, rid, attempt, what): + self.load() + return (int(rid), int(attempt), what) in self.marks() + + def count(self): + """Re-runs this sweep asked for in the last window_hours, from the issue's markers.""" + self.load() + since = self.now() - self.hours * 3600 + return sum(1 for (_, _, what), at in self.marks().items() if what == 'rerun' and (stamp(at) or 0) >= since) + + def where(self): + return f'issue #{self.issue}' if self.issue else 'no tracking issue yet' + + def _open(self): + n = self.cfg + got = self.gh.call('issues', method='POST', data={ + 'title': n['title'], 'labels': n['labels'], + 'body': ('Each comment below is one automatic re-run of a failed run (or, in fleet mode, a run that ' + 'would have had one), with its jobs, the signs and the memory pressure numbers. Many of them ' + 'mean a machine that stalls or fills up: fix that, the re-run only hides it for one attempt. ' + 'Closing this issue is fine; the next re-run opens a new one. Rules and runbook: ' + '`.github/fork-ci-rerun.md`.')}) + mine = int(got['number']) + # A dispatched pass may run beside a scheduled one (fork-ci-rerun.yml); if both opened one, the lowest wins + # and the other is closed, pointing at it, so there is only ever one. + ours = self._ours() + if ours and ours[0] < mine: + self.gh.call(f'issues/{mine}/comments', method='POST', data={'body': f'Duplicate of #{ours[0]}.'}) + self.gh.call(f'issues/{mine}', method='PATCH', data={'state': 'closed', 'state_reason': 'not_planned'}) + return ours[0] + return mine + + def add(self, run, what, text): + """Write one comment for this run's attempt; returns the issue's number.""" + self.load() + rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) + url = run.get('html_url') or f'run {rid}' + head = (f'**Re-run asked** for [run {rid}]({url}) attempt {attempt}: GitHub\'s "re-run failed jobs" makes ' + f'attempt {attempt + 1}; if the run has no attempt {attempt + 1}, the request did not go through ' + '(the sweep\'s log says why).' if what == 'rerun' else + f'**Not re-run (fleet mode)**: [run {rid}]({url}) attempt {attempt} qualifies for its one automatic ' + 're-run; re-run it by hand.') + body = (f'\n{head}\n\n' + f'{run.get("event")} `{run.get("head_branch")}` `{str(run.get("head_sha"))[:12]}`\n\n' + + ''.join(f'- {part.strip()}\n' for part in text.split(' | ')))[:LONGEST] + if self.issue is None: + self.issue = self._open() + got = self.gh.call(f'issues/{self.issue}/comments', method='POST', data={'body': body}) + self.comments.append(dict(got if isinstance(got, dict) else {}, body=body, + created_at=(got or {}).get('created_at') or iso(self.now()))) + return self.issue + + +def rerun(gh, run, record): + """GitHub's own re-run of failed jobs (and the jobs after them), once: (True, how) when this pass asked for it. + record() writes the durable notice first, so a reply lost after GitHub took the request still leaves the why.""" rid, attempt = int(run['id']), int(run.get('run_attempt') or 1) fresh = gh.call(f'actions/runs/{rid}') if int(fresh.get('run_attempt') or 1) != attempt or fresh.get('status') != 'completed': return False, f'already re-run or running again (attempt {fresh.get("run_attempt")}, {fresh.get("status")})' + record() try: gh.call(f'actions/runs/{rid}/rerun-failed-jobs', method='POST') except urllib.error.HTTPError as e: @@ -349,13 +560,19 @@ def cell(text): def sweep(cfg, env, gh, now=time.time, run_id=None, out=None): - if run_id: - runs = [gh.call(f'actions/runs/{int(run_id)}')] - else: - since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(now() - cfg['window_hours'] * 3600)) - runs = gh.pages(f'actions/workflows/{cfg["workflow"]}/runs?created=%3E%3D{since}', 'workflow_runs', most=3) - recent = now() - cfg['catch_up_minutes'] * 60 + notice = Notice(cfg, gh, now) rows, trouble, asked = [], 0, 0 + try: + if run_id: + runs = [gh.call(f'actions/runs/{int(run_id)}')] + else: + since = iso(now() - cfg['list_hours'] * 3600) + runs = gh.pages(f'actions/workflows/{cfg["workflow"]}/runs?created=%3E%3D{since}', 'workflow_runs') + except TROUBLE as e: + runs, trouble = [], 1 + say(f'::warning::fork-ci re-run: the runs: {e.__class__.__name__}: {e}; the next pass tries again', out) + rows.append(f'| - | not read | {cell(e.__class__.__name__)}: {cell(e)} |') + recent = now() - cfg['catch_up_minutes'] * 60 for run in runs: rid = run.get('id') try: @@ -363,25 +580,36 @@ def sweep(cfg, env, gh, now=time.time, run_id=None, out=None): continue # ended before the catch-up window: too old to re-run now what, text = decide(cfg, gh, run, env) where = f'run {rid} ({run.get("event")} {run.get("head_branch")} {str(run.get("head_sha"))[:12]})' + attempt = int(run.get('run_attempt') or 1) if what == 'quiet': continue if what == 'leave': say(f'{where}: left red: {text}', out) rows.append(f'| {rid} | left red | {cell(text)} |') continue + notice.load(since=stamp(run.get('created_at'))) if what == 'report': skip = cfg['skip_when'] - say(f'::error title=fork-ci re-run::{where} qualifies for its one automatic re-run, but ' - f'{skip["variable"]}={env.get(skip["variable"])}: not re-run; re-run it by hand ' - f'(.github/fork-ci-rerun.md). {text}', out) - rows.append(f'| {rid} | not re-run ({skip["variable"]}={cell(env.get(skip["variable"]))}) ' - f'| {cell(text)} |') + state = f'{skip["variable"]}={env.get(skip["variable"])}' + if notice.has(rid, attempt, 'report'): + say(f'{where}: qualifies, not re-run ({state}); reported already ({notice.where()})', out) + rows.append(f'| {rid} | not re-run ({cell(state)}), reported already | {cell(text)} |') + continue + issue = notice.add(run, 'report', text) + say(f'::error title=fork-ci re-run::{where} qualifies for its one automatic re-run, but {state}: ' + f'not re-run; re-run it by hand (.github/fork-ci-rerun.md, issue #{issue}). {text}', out) + rows.append(f'| {rid} | not re-run ({cell(state)}), issue #{issue} | {cell(text)} |') continue - done, how = rerun(gh, run) + + def record(): + if not notice.has(rid, attempt, 'rerun'): + notice.add(run, 'rerun', text) + done, how = rerun(gh, run, record) if done: asked += 1 - say(f'::warning title=fork-ci re-run::{where} re-run once ({how}), infrastructure: {text}', out) - rows.append(f'| {rid} | **re-run** ({cell(how)}) | {cell(text)} |') + say(f'::warning title=fork-ci re-run::{where} re-run once ({how}, {notice.where()}), ' + f'infrastructure: {text}', out) + rows.append(f'| {rid} | **re-run** ({cell(how)}, {notice.where()}) | {cell(text)} |') else: say(f'{where}: {how}', out) rows.append(f'| {rid} | {cell(how)} | {cell(text)} |') @@ -389,11 +617,12 @@ def sweep(cfg, env, gh, now=time.time, run_id=None, out=None): trouble += 1 say(f'::warning::fork-ci re-run: run {rid}: {e.__class__.__name__}: {e}; the next pass tries again', out) rows.append(f'| {rid} | not read | {cell(e.__class__.__name__)}: {cell(e)} |') - bot = [r for r in runs if int(r.get('run_attempt') or 1) > 1 and login(r.get('triggering_actor')) - not in cfg['actors']] - count = (f'{asked} run(s) re-run by this pass; {len(bot)} run(s) whose latest attempt was an automatic re-run ' - f'(this script\'s or the fleet sweep\'s) in the last {cfg["window_hours"]} h' - + (': ' + ', '.join(str(r.get('id')) for r in bot) if bot else '')) + try: + count = (f'{asked} run(s) re-run by this pass; {notice.count()} automatic re-run(s) asked in the last ' + f'{cfg["window_hours"]} h ({notice.where()})') + except TROUBLE as e: + trouble += 1 + count = f'{asked} run(s) re-run by this pass; the tracking issue could not be read ({e.__class__.__name__})' say(f'fork-ci re-run: {count}', out) lines = ['### fork-ci re-run', '', count, ''] if rows: @@ -412,7 +641,7 @@ def main(argv=None, env=os.environ): if a.verb == 'check': say(f'{a.config}: ok') return 0 - return sweep(cfg, env, GitHub(env.get('GITHUB_TOKEN'), cfg['repository']), run_id=a.run_id) + return sweep(cfg, env, GitHub(env.get('GITHUB_TOKEN'), cfg['fleet']['repository']), run_id=a.run_id) if __name__ == '__main__': diff --git a/.github/scripts/fork-ci-rerun.test.py b/.github/scripts/fork-ci-rerun.test.py index 876287039c2f7..2eb8056ea8ce4 100644 --- a/.github/scripts/fork-ci-rerun.test.py +++ b/.github/scripts/fork-ci-rerun.test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -"""Tests of fork-ci-rerun.py (the one automatic re-run of an infrastructure failure), with a fake GitHub, and of its -wiring in fork-ci-rerun.yml. Run: python3 .""" -import copy, io, json, os, re, subprocess, sys, tempfile, time, unittest, urllib.error +"""Tests of fork-ci-rerun.py (the one automatic re-run of an infrastructure failure), with a fake GitHub that pages +like the real one (the script's own GitHub.pages reads it), and of its wiring in fork-ci-rerun.yml. Run: python3 .""" +import copy, io, json, os, re, shutil, subprocess, sys, tempfile, time, unittest, urllib.error, urllib.parse import importlib.util sys.dont_write_bytecode = True # importing the script must not leave __pycache__ in the tree @@ -17,34 +17,48 @@ _spec.loader.exec_module(probe) CFG = cr.load() -REPO = CFG['repository'] -OWNER = CFG['actors'][0] -BRANCH = CFG['push_branches'][0] +FLEET_CFG = CFG['fleet'] +REPO = FLEET_CFG['repository'] +OWNER = FLEET_CFG['trusted']['actors'][0] +BRANCH = FLEET_CFG['trusted']['push_branches'][0] FOLLOWER = CFG['followers'][0] +END_STEP = CFG['probe_end_step'] SKIP_VAR, SKIP_VALUE = CFG['skip_when']['variable'], CFG['skip_when']['is'][0] -BOT = 'github-actions[bot]' +TITLE = CFG['notice']['title'] +BOT = CFG['notice']['author'] +ACTIONS = 'github-actions' # the triggering actor of GitHub's re-run from a workflow token +HOSTED = CFG['hosted_runner_groups'][0] T0 = 1_800_000_000 SHA = '0123456789abcdef0123456789abcdef01234567' NEWER = 'fedcba9876543210fedcba9876543210fedcba98' -# GitHub's own messages, as its annotations carry them. +# The fleet's trust rule asks git whether the upstream sync made a pushed commit; here, a set. +SYNCED = set() +cr.FLEET.made_by_the_sync = lambda cfg, sha, run=None: (f'commit {sha[:12]} was pushed by the upstream sync' + if sha in SYNCED else None) + +# GitHub's own messages, word for word as its annotations carry them. LOST = ('The self-hosted runner: fleet-vm-1 lost communication with the server. Verify the machine is running and ' 'has a healthy network connection. Anything in your workflow that terminates the runner process, starves it ' 'for CPU/Memory, or blocks its network access can cause this error.') +HOSTED_LOST = 'The hosted runner: GitHub Actions 7 lost communication with the server. Anything in your workflow...' SHUTDOWN = 'The runner has received a shutdown signal. This can happen when the runner service is stopped.' -JOB_LIMIT = 'The job has exceeded the maximum execution time of 15m0s' +JOB_LIMIT = 'The job running on runner fleet-vm-1 has exceeded the maximum execution time of 15 minutes.' STEP_LIMIT = "The action 'Test' has timed out after 10 minutes." +CANCEL_BY = f'The run was canceled by @{OWNER}.' CANCELED = 'The operation was canceled.' EXIT_1 = 'Process completed with exit code 1.' EXIT_137 = 'Process completed with exit code 137.' +MEMORY_WORDS = 'OOM kill: memory.max hit, memory full, the box stalled (full) 90% of the job' def iso(t): return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(t)) -def ann(message, level='failure', title=''): - return {'message': message, 'annotation_level': level, 'title': title} +def ann(message, level='failure', title='', path='.github'): + """An annotation as GitHub hands it back. A step's ::error::/::warning:: has the same fields as the runner's own.""" + return {'message': message, 'annotation_level': level, 'title': title, 'path': path} def note(host=None, cgroups=None, status='failure', seconds=600): @@ -65,36 +79,100 @@ def box(**kw): CALM = note() +FULL = note(cgroups=[box(oom_kill=1, full_share=30.0)]) # the box stalled and an OOM kill in it +STALLED = note(host={'full_share': 25.0}) -def run(rid=1, attempt=1, conclusion='failure', status='completed', event='push', branch=BRANCH, actor=OWNER, - by=OWNER, head_repo=REPO, updated=T0 - 120, workflow='fork-tests.yml'): - return {'id': rid, 'run_attempt': attempt, 'status': status, 'conclusion': conclusion, 'event': event, - 'head_branch': branch, 'head_sha': SHA, 'path': f'.github/workflows/{workflow}', - 'repository': {'full_name': REPO}, 'head_repository': {'full_name': head_repo}, - 'actor': {'login': actor}, 'triggering_actor': {'login': by}, 'updated_at': iso(updated)} +def step(name, conclusion='success', status='completed', took=5, at=0): + return {'name': name, 'status': status, 'conclusion': conclusion, + 'started_at': iso(T0 - 3600 + at), 'completed_at': iso(T0 - 3600 + at + took) if status == 'completed' + else None} + + +def ran(test='failure', took=60, end=True): + """A job that reached its end step (the note is its own).""" + return [step('Checkout'), step('Note memory pressure at the start', at=5), step('Test', test, took=took, at=10), + step(END_STEP, at=10 + took) if end else step(END_STEP, None, 'queued')] + +def died(): + """A job whose runner went away mid-test: the test step never ended, the end step never ran.""" + return [step('Checkout'), step('Note memory pressure at the start', at=5), + step('Test', None, 'in_progress', at=10), step(END_STEP, None, 'queued')] -def job(jid, name='Test (1)', conclusion='failure'): + +def run(rid=1, attempt=1, conclusion='failure', status='completed', event='push', branch=BRANCH, actor=OWNER, + by=OWNER, repo=REPO, head_repo=REPO, updated=T0 - 120, created=T0 - 3600, workflow='fork-tests.yml', pr=7): + r = {'id': rid, 'run_attempt': attempt, 'status': status, 'conclusion': conclusion, 'event': event, + 'head_branch': branch, 'head_sha': SHA, 'path': f'.github/workflows/{workflow}', + 'html_url': f'https://github.example/{repo}/actions/runs/{rid}', + 'repository': {'full_name': repo}, 'head_repository': {'full_name': head_repo}, + 'actor': {'login': actor}, 'triggering_actor': {'login': by}, 'updated_at': iso(updated), + 'created_at': iso(created), 'pull_requests': []} + if event == 'pull_request' and head_repo == repo: + r['pull_requests'] = [{'number': pr}] # GitHub lists none for a pull request from another fork + return r + + +def job(jid, name='Test (1)', conclusion='failure', steps=None, group='fleet', runner='fleet-vm-1', took=600): return {'id': jid, 'name': name, 'conclusion': conclusion, 'status': 'completed', + 'steps': steps if steps is not None else ran(), 'runner_group_name': group, 'runner_name': runner, + 'started_at': iso(T0 - 3600), 'completed_at': iso(T0 - 3600 + took), 'check_run_url': f'https://api.example/repos/{REPO}/check-runs/{jid}'} class Fake: - """GitHub's REST API as fork-ci-rerun.py uses it; a re-run makes a new attempt triggered by github-actions.""" + """GitHub's REST API as fork-ci-rerun.py uses it. Listings page like GitHub's (page/per_page), read by the script's + own GitHub.pages; a re-run makes a new attempt triggered by github-actions; issues and comments are kept.""" - def __init__(self, runs, jobs=None, annotations=None, heads=None): + pages = cr.GitHub.pages # the real paging code, over this fake's call() + + def __init__(self, runs, jobs=None, annotations=None, heads=None, pulls=None): self.runs = {r['id']: r for r in runs} self.jobs = jobs or {} self.ann = annotations or {} self.heads = heads if heads is not None else {BRANCH: SHA, 'feature': SHA} + self.pulls = pulls if pulls is not None else {7: {'head': {'repo': {'full_name': REPO}}, + 'base': {'repo': {'full_name': REPO}}, + 'user': {'login': OWNER}}} + self.issues = [] self.attempts = {} self.calls = [] - self.post_fails = None # an exception to raise for the next POST - self.lost_reply = False # the POST reaches GitHub, its reply never comes back + self.now = T0 + self.post_fails = None # an exception to raise for the next re-run POST + self.lost_reply = False # the re-run POST reaches GitHub, its reply never comes back + self.comment_fails = False # a comment POST fails + self.on_read = None # called with the run before GET actions/runs/N answers def call(self, path, method='GET', data=None): self.calls.append((method, path)) + base, _, query = path.partition('?') + q = urllib.parse.parse_qs(query) + + def paged(items): + if 'page' not in q: + return copy.deepcopy(items) + n, per = int(q['page'][0]), int(q['per_page'][0]) + return copy.deepcopy(items[(n - 1) * per:n * per]) + + if method == 'POST' and base == 'issues': + number = max([i['number'] for i in self.issues] + [40]) + 1 + self.issues.append({'number': number, 'title': data['title'], 'labels': data['labels'], + 'body': data['body'], 'state': 'open', 'user': {'login': BOT}, 'comments': []}) + return {'number': number} + m = re.fullmatch(r'issues/(\d+)/comments', base) + if m and method == 'POST': + if self.comment_fails: + raise urllib.error.URLError('comment: reset') + issue = self.issue(int(m[1])) + c = {'id': 9000 + sum(len(i['comments']) for i in self.issues), 'body': data['body'], + 'user': {'login': BOT}, 'created_at': iso(self.now)} + issue['comments'].append(c) + return copy.deepcopy(c) + m = re.fullmatch(r'issues/(\d+)', base) + if m and method == 'PATCH': + self.issue(int(m[1])).update(data) + return {} if method == 'POST': m = re.fullmatch(r'actions/runs/(\d+)/rerun-failed-jobs', path) if not m: @@ -107,47 +185,71 @@ def call(self, path, method='GET', data=None): raise urllib.error.HTTPError(path, 403, 'This workflow is already running', {}, None) self.attempts[(r['id'], r['run_attempt'])] = copy.deepcopy(r) r.update(run_attempt=r['run_attempt'] + 1, status='queued', conclusion=None, - triggering_actor={'login': BOT}) + triggering_actor={'login': ACTIONS}) if self.lost_reply: raise urllib.error.URLError('connection reset') return {} - m = re.fullmatch(r'actions/runs/(\d+)', path) + if method != 'GET': + raise AssertionError(f'unexpected {method} {path}') + if base == f'actions/workflows/{CFG["workflow"]}/runs': + since = cr.stamp(urllib.parse.unquote(q['created'][0]).lstrip('>=')) + listed = sorted((r for r in self.runs.values() if cr.stamp(r['created_at']) >= since), + key=lambda r: -r['id']) # newest first, as GitHub lists them + return {'total_count': len(listed), 'workflow_runs': paged(listed)} + m = re.fullmatch(r'actions/runs/(\d+)/jobs', base) if m: + return {'jobs': paged(self.jobs.get(int(m[1]), []))} + m = re.fullmatch(r'actions/runs/(\d+)', base) + if m: + if self.on_read: + self.on_read(self.runs[int(m[1])]) return copy.deepcopy(self.runs[int(m[1])]) - m = re.fullmatch(r'actions/runs/(\d+)/attempts/(\d+)', path) + m = re.fullmatch(r'actions/runs/(\d+)/attempts/(\d+)', base) if m: got = self.attempts.get((int(m[1]), int(m[2]))) if got is None: got = dict(self.runs[int(m[1])], run_attempt=int(m[2]), triggering_actor={'login': OWNER}) return copy.deepcopy(got) - m = re.fullmatch(r'check-runs/(\d+)/annotations\?per_page=100', path) + m = re.fullmatch(r'check-runs/(\d+)/annotations', base) if m: - return copy.deepcopy(self.ann.get(int(m[1]), [])) - m = re.fullmatch(r'branches/(.+)', path) + return paged(self.ann.get(int(m[1]), [])) + m = re.fullmatch(r'branches/(.+)', base) if m: if m[1] not in self.heads: raise urllib.error.HTTPError(path, 404, 'Branch not found', {}, None) return {'commit': {'sha': self.heads[m[1]]}} + m = re.fullmatch(r'pulls/(\d+)', base) + if m: + return copy.deepcopy(self.pulls[int(m[1])]) + if base == 'issues': + assert q['state'] == ['open'] and q['creator'] == [BOT], q + # The script checks the author itself: here the creator filter is not applied. + listed = [{k: v for k, v in i.items() if k != 'comments'} for i in self.issues if i['state'] == 'open'] + return paged(listed) + m = re.fullmatch(r'issues/(\d+)/comments', base) + if m: + since = cr.stamp(q['since'][0]) + # GitHub's `since` is by updated_at: an old comment edited lately is listed too. + return paged([c for c in self.issue(int(m[1]))['comments'] + if cr.stamp(c.get('updated_at') or c['created_at']) >= since]) raise AssertionError(f'unexpected GET {path}') - def pages(self, path, key, most=10): - self.calls.append(('GET', path)) - if path.startswith(f'actions/workflows/{CFG["workflow"]}/runs?created=%3E%3D'): - return [copy.deepcopy(r) for r in self.runs.values()] - m = re.fullmatch(r'actions/runs/(\d+)/jobs\?filter=latest', path) - if m: - return copy.deepcopy(self.jobs.get(int(m[1]), [])) - raise AssertionError(f'unexpected pages {path}') + def issue(self, number): + return next(i for i in self.issues if i['number'] == number) def posts(self): - return [p for m, p in self.calls if m == 'POST'] + return [p for m, p in self.calls if m == 'POST' and p.startswith('actions/')] + + def comments(self): + return [c['body'] for i in self.issues for c in i['comments']] def sweep(gh, env=None, run_id=None, cfg=CFG): out = io.StringIO() with tempfile.TemporaryDirectory() as d: path = os.path.join(d, 'summary.md') - code = cr.sweep(cfg, dict(env or {}, GITHUB_STEP_SUMMARY=path), gh, now=lambda: T0, run_id=run_id, out=out) + code = cr.sweep(cfg, dict(env or {}, GITHUB_STEP_SUMMARY=path), gh, now=lambda: gh.now, run_id=run_id, + out=out) text = '' if os.path.exists(path): with open(path, encoding='utf-8') as f: @@ -155,37 +257,45 @@ def sweep(gh, env=None, run_id=None, cfg=CFG): return code, out.getvalue(), text -def one(annotations, job_conclusion='failure', **kw): +def one(annotations, job_kw=None, **kw): """A failed run whose one failed job (id 11) carries these annotations, plus a passed job and the follower.""" - jobs = [job(11, conclusion=job_conclusion), job(12, 'Lint', 'success'), job(13, FOLLOWER, 'failure')] + jobs = [job(11, **(job_kw or {})), job(12, 'Lint', 'success'), job(13, FOLLOWER, 'failure', steps=[])] return Fake([run(**kw)], {1: jobs}, {11: annotations, 13: [ann(EXIT_1)]}) -class Signs(unittest.TestCase): +def lost(**kw): + return one([ann(LOST), ann(CANCELED)], dict(steps=died()), **kw) + + +class Rerun(unittest.TestCase): def assertReran(self, gh, out): self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) self.assertIn('::warning title=fork-ci re-run::run 1', out) + def assertLeft(self, gh, code, out, why=None): + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn('left red', out) + if why: + self.assertIn(why, out) + + +class Signs(Rerun): def test_lost_runner_is_rerun_once_with_the_message_in_the_log(self): - for message in (LOST, SHUTDOWN, 'The hosted runner: GitHub Actions 7 lost communication with the server.'): - gh = one([ann(message), ann(CANCELED)]) - code, out, text = sweep(gh) - self.assertEqual(code, 0, out) - self.assertReran(gh, out) - self.assertIn('lost', out) - self.assertIn('**re-run**', text) - - def test_lost_runner_needs_no_pressure_note_an_old_run_without_the_new_lines(self): - gh = one([ann(LOST)]) # the job never reached its end step: no note - self.assertReran(gh, sweep(gh)[1]) + for message, runner in ((LOST, 'fleet-vm-1'), (SHUTDOWN, 'fleet-vm-1'), (HOSTED_LOST, 'GitHub Actions 7')): + with self.subTest(message[:30]): + gh = one([ann(message), ann(CANCELED)], dict(steps=died(), runner=runner)) + code, out, text = sweep(gh) + self.assertEqual(code, 0, out) + self.assertReran(gh, out) + self.assertIn('lost', out) + self.assertIn('**re-run**', text) def test_a_full_box_is_rerun_with_its_numbers_in_the_log(self): cases = { - 'box OOM kill': note(cgroups=[box(oom_kill=1)]), - 'box memory.max hits': note(cgroups=[box(max=3)]), - 'machine OOM kill': note(host={'oom_kill': 2}), - 'memory.high throttling while stalled': note(host={'full_share': 25.0}, cgroups=[box(high=40)]), - 'memory.high throttling, the box stalled': note(cgroups=[box(high=40, full_share=30.0)]), + 'box OOM kills while it stalled': FULL, + 'box memory.max hits while it stalled': note(cgroups=[box(max=3, full_share=12.0)]), + 'memory.high throttling while the box stalled': note(cgroups=[box(high=40, full_share=30.0)]), + 'a box with only a high limit': note(cgroups=[box(max_mb=None, oom_kill=2, full_share=15.0)]), } for label, n in cases.items(): with self.subTest(label): @@ -194,302 +304,591 @@ def test_a_full_box_is_rerun_with_its_numbers_in_the_log(self): self.assertReran(gh, out) self.assertIn('memory full', out) self.assertRegex(out, r'machine: full [0-9.]+% some [0-9.]+% of 600 s') - self.assertIn('box /fleet.slice/job-1.service: max 8192 MB high 6144 MB', out) + self.assertIn('box /fleet.slice/job-1.service: max', out) self.assertIn('memory full', text) - def test_memory_high_events_without_a_stall_are_no_sign(self): - gh = one([ann(EXIT_1), note(host={'full_share': 2.0}, cgroups=[box(high=40, full_share=3.0)])]) - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), [], out) - self.assertIn('no infrastructure sign', out) + def test_one_event_or_a_stall_alone_is_no_memory_sign(self): + cases = { + 'one OOM kill, the box not stalled': note(cgroups=[box(oom_kill=1, full_share=2.0)]), + 'one memory.max hit, not stalled': note(cgroups=[box(max=1)]), + 'memory.high throttling, not stalled': note(host={'full_share': 2.0}, cgroups=[box(high=40)]), + 'an OOM kill on the machine, not in the box': note(host={'oom_kill': 3, 'full_share': 50.0}), + 'the machine stalled, the box had no event': note(host={'full_share': 50.0}, + cgroups=[box(full_share=50.0)]), + 'a cgroup with no memory limit': note(cgroups=[box(max_mb=None, high_mb=None, oom_kill=4, + full_share=80.0)]), + 'numbers that are not numbers': note(cgroups=[box(oom_kill=True, max='9', full_share='90')]), + } + for label, n in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), n]) + self.assertLeft(gh, *sweep(gh)[:2], 'no infrastructure sign') + + def test_an_oom_kill_on_a_github_hosted_runner_never_qualifies(self): + for group in (HOSTED, None, ''): + with self.subTest(group=group): + gh = one([ann(EXIT_137), FULL], dict(group=group)) + self.assertLeft(gh, *sweep(gh)[:2]) + + def test_a_memory_sign_needs_the_job_and_its_note_to_have_failed(self): + for kw, status in ((dict(conclusion='cancelled'), 'failure'), ({}, 'cancelled'), ({}, 'success')): + with self.subTest(kw=kw, status=status): + n = note(cgroups=[box(oom_kill=1, full_share=30.0)], status=status) + gh = one([ann(EXIT_137), n], kw) + self.assertLeft(gh, *sweep(gh)[:2]) def test_time_limit_while_stalled_is_rerun(self): - for message in (JOB_LIMIT, STEP_LIMIT): - for stall in ({'full_share': 12.0}, {'full_avg300': 15.0}): - with self.subTest(message=message, stall=stall): - gh = one([ann(message), ann(CANCELED), note(host=stall, status='cancelled')], 'cancelled', - conclusion='cancelled') - code, out, _ = sweep(gh) - self.assertReran(gh, out) - self.assertIn('limit while stalled', out) - gh = one([ann(JOB_LIMIT), note(cgroups=[box(full_share=50.0)], status='cancelled')], 'cancelled') + for stall in ({'full_share': 12.0}, {'full_avg300': 15.0}): + with self.subTest(stall=stall): + gh = one([ann(JOB_LIMIT), ann(CANCELED), note(host=stall, status='cancelled')], + dict(conclusion='cancelled', steps=ran('cancelled', took=880), took=900)) + code, out, _ = sweep(gh) + self.assertReran(gh, out) + self.assertIn('limit while stalled', out) + gh = one([ann(STEP_LIMIT), note(cgroups=[box(full_share=50.0)])], dict(steps=ran('failure', took=600))) + self.assertReran(gh, sweep(gh)[1]) + + def test_a_timed_out_run_is_a_candidate(self): + gh = one([ann(JOB_LIMIT), STALLED], dict(conclusion='timed_out', steps=ran('cancelled', took=880), took=900), + conclusion='timed_out') self.assertReran(gh, sweep(gh)[1]) + def test_a_time_limit_message_needs_the_time_really_spent(self): + cases = { + 'the job ran 3 of its 15 minutes': (JOB_LIMIT, dict(steps=ran('cancelled', took=170), took=180)), + 'the step ran 1 of its 10 minutes': (STEP_LIMIT, dict(steps=ran('failure', took=60))), + 'the step it names never ran': ("The action 'Other' has timed out after 10 minutes.", + dict(steps=ran('failure', took=600))), + 'another step failed': (JOB_LIMIT, dict(steps=ran('failure', took=880), took=900)), + } + for label, (message, kw) in cases.items(): + with self.subTest(label): + gh = one([ann(message), STALLED], kw) + self.assertLeft(gh, *sweep(gh)[:2]) + + def test_a_stalled_note_without_its_status_is_no_limit_sign(self): + n = dict(STALLED, message=STALLED['message'].replace('"status":"failure",', '')) + self.assertNotIn('status', n['message']) + gh = one([ann(STEP_LIMIT), n], dict(steps=ran('failure', took=600))) + self.assertLeft(gh, *sweep(gh)[:2], 'garbled') + def test_time_limit_on_a_calm_machine_is_left_red(self): - gh = one([ann(JOB_LIMIT), ann(CANCELED), CALM], 'cancelled') + gh = one([ann(JOB_LIMIT), ann(CANCELED), CALM], dict(conclusion='cancelled', steps=ran('cancelled', took=880), + took=900)) code, out, text = sweep(gh) - self.assertEqual(gh.posts(), [], out) - self.assertIn('not stalled', out) + self.assertLeft(gh, code, out, 'not stalled') self.assertIn('left red', text) def test_time_limit_without_a_note_is_left_red_fail_closed(self): - gh = one([ann(JOB_LIMIT), ann(CANCELED)], 'cancelled') # an old run, or the note was dropped - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), [], out) - self.assertIn('cannot tell it was stalled', out) + gh = one([ann(JOB_LIMIT), ann(CANCELED)], dict(conclusion='cancelled', took=900, + steps=ran('cancelled', took=880, end=False))) + self.assertLeft(gh, *sweep(gh)[:2], 'cannot tell it was stalled') def test_a_test_failure_is_never_rerun(self): gh = one([ann(EXIT_1), CALM]) code, out, text = sweep(gh) - self.assertEqual((code, gh.posts()), (0, []), out) - self.assertIn('Test (1) ended failure: no infrastructure sign', out) + self.assertLeft(gh, code, out, 'Test (1) ended failure: no infrastructure sign') self.assertIn('| 1 | left red |', text) - def test_a_job_with_no_readable_signs_is_left_red(self): - garbled = dict(CALM, message=cr.NOTE_HEAD + '{"host": ') - cases = {'no annotations at all': [], 'a garbled note': [ann(EXIT_1), garbled], - 'a note without its head': [ann(EXIT_1), dict(CALM, message='{"host":{},"cgroups":[]}')], - 'a note of the wrong shape': [ann(EXIT_1), dict(CALM, message=cr.NOTE_HEAD + '[1, 2]')], - 'two notes (one forged)': [ann(EXIT_137), CALM, note(cgroups=[box(oom_kill=5)])], - 'numbers that are not numbers': [ann(EXIT_137), note(host={'oom_kill': '9'}, - cgroups=[box(oom_kill=True, max='x')])], - 'a cancel by a person': [ann(CANCELED)]} - for label, anns in cases.items(): - with self.subTest(label): - gh = one(anns) - code, out, _ = sweep(gh) - self.assertEqual((code, gh.posts()), (0, []), out) - self.assertIn('left red', out) - def test_one_job_without_a_sign_leaves_the_whole_run_red(self): - jobs = [job(11), job(12, 'Test (2)'), job(13, FOLLOWER)] + jobs = [job(11, steps=died()), job(12, 'Test (2)'), job(13, FOLLOWER, steps=[])] gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_1), CALM]}) - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), [], out) - self.assertIn('Test (2) ended failure', out) + self.assertLeft(gh, *sweep(gh)[:2], 'Test (2) ended failure') def test_every_failed_job_with_a_sign_is_rerun_together(self): - jobs = [job(11), job(12, 'Build and Typecheck'), job(13, FOLLOWER)] - gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_137), note(cgroups=[box(oom_kill=1)])]}) + jobs = [job(11, steps=died()), job(12, 'Build and Typecheck'), job(13, FOLLOWER, steps=[])] + gh = Fake([run()], {1: jobs}, {11: [ann(LOST)], 12: [ann(EXIT_137), FULL]}) code, out, _ = sweep(gh) self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) self.assertIn('Test (1) (failure): lost', out) self.assertIn('Build and Typecheck (failure): memory full', out) def test_only_a_follower_failed_is_left_red(self): - gh = Fake([run()], {1: [job(12, 'Lint', 'success'), job(13, FOLLOWER)]}, {13: [ann(LOST)]}) - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), [], out) - self.assertIn('no failed job of its own', out) + gh = Fake([run()], {1: [job(12, 'Lint', 'success'), job(13, FOLLOWER, steps=died())]}, {13: [ann(LOST)]}) + self.assertLeft(gh, *sweep(gh)[:2], 'no failed job of its own') def test_a_job_skipped_after_a_failed_one_needs_no_sign(self): - jobs = [job(11), job(12, 'Lint', 'skipped'), job(13, FOLLOWER)] + jobs = [job(11, steps=died()), job(12, 'Lint', 'skipped'), job(13, FOLLOWER, steps=[])] gh = Fake([run()], {1: jobs}, {11: [ann(LOST)]}) - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + self.assertReran(gh, sweep(gh)[1]) + + +class StepOutputIsNoSign(Rerun): + """A step can print any annotation (::error::GitHub's words, ::notice title=fork-ci-pressure::...), with exactly + the fields the runner's own have. Text is never enough: the job's own record must agree.""" + + FORGED = note(host={'full_share': 90.0, 'oom_kill': 3}, cgroups=[box(oom_kill=3, max=9, full_share=90.0)]) + + def test_a_failing_test_that_prints_the_lost_text_and_the_memory_words_is_not_rerun(self): + cases = { + 'a failed test step': ([ann(LOST), ann(MEMORY_WORDS), ann(EXIT_1), CALM], dict(steps=ran())), + 'a failed test step, the end step never ran': ([ann(LOST), ann(MEMORY_WORDS), ann(EXIT_1)], + dict(steps=ran(end=False))), + 'no failed step, but the end step ran (the runner lived)': ([ann(LOST), CALM], + dict(steps=ran('success'))), + 'then a hang into the time limit': ([ann(LOST), ann(JOB_LIMIT)], + dict(steps=ran('cancelled', took=880, end=False), took=900)), + 'on another runner': ([ann(LOST)], dict(steps=died(), runner='fleet-vm-2')), + 'a job that ended cancelled': ([ann(LOST)], dict(steps=died(), conclusion='cancelled')), + 'inside a longer line': ([ann('Error: ' + LOST)], dict(steps=died())), + 'as a warning': ([ann(LOST, 'warning')], dict(steps=died())), + 'with a title': ([ann(LOST, title='Error')], dict(steps=died())), + 'on a file': ([ann(LOST, path='src/x.test.ts')], dict(steps=died())), + 'a forged note beside the real one': ([ann(LOST), ann(EXIT_137), self.FORGED, CALM], dict(steps=ran())), + } + for label, (anns, kw) in cases.items(): + with self.subTest(label): + gh = one(anns, kw) + self.assertLeft(gh, *sweep(gh)[:2]) + self.assertEqual(gh.comments(), []) + + def test_a_note_counts_only_when_the_job_wrote_it_and_it_is_the_only_one(self): + """Each rule of note_of, pinned: the same note qualifies alone, and each bad variant does not.""" + good = one([ann(EXIT_137), self.FORGED]) + self.assertReran(good, sweep(good)[1]) + garbled = dict(self.FORGED, message=cr.NOTE_HEAD + '{"host": ') + wrong_shape = json.loads(self.FORGED['message'][len(cr.NOTE_HEAD):]) + cases = { + 'two notes, the forged one first': [self.FORGED, CALM], + 'two notes, the forged one last': [CALM, self.FORGED], + 'two forged notes': [self.FORGED, self.FORGED], + 'a note without its header': [dict(self.FORGED, message=self.FORGED['message'][len(cr.NOTE_HEAD):])], + 'another version\'s header': [dict(self.FORGED, message=self.FORGED['message'].replace(' v1 ', ' v0 ', 1))], + 'a garbled note': [garbled], + 'cgroups not a list': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps( + dict(wrong_shape, cgroups={'a': box(oom_kill=3, full_share=90.0)})))], + 'a box not an object': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps( + dict(wrong_shape, cgroups=['oom_kill']), separators=(',', ':')))], + 'host not an object': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps(dict(wrong_shape, host=[])))], + 'no status': [dict(self.FORGED, message=cr.NOTE_HEAD + json.dumps( + {k: v for k, v in wrong_shape.items() if k != 'status'}))], + 'a list, not an object': [dict(self.FORGED, message=cr.NOTE_HEAD + '[1, 2]')], + 'at error level': [dict(self.FORGED, annotation_level='failure')], + 'on a file': [dict(self.FORGED, path='src/x.ts')], + } + for label, notes in cases.items(): + with self.subTest(label): + gh = one([ann(EXIT_137)] + notes) + code, out, _ = sweep(gh) + self.assertLeft(gh, code, out) + for label, steps in {'the end step never ran': ran(end=False), + 'the end step failed': ran()[:-1] + [step(END_STEP, 'failure')], + 'no steps listed': []}.items(): + with self.subTest(label): + gh = one([ann(EXIT_137), self.FORGED], dict(steps=steps)) + self.assertLeft(gh, *sweep(gh)[:2], 'did not complete') + + +class Cancel(Rerun): + def test_a_run_a_person_cancelled_never_qualifies(self): + cases = { + 'a cancel while the box was full (an OOM elsewhere)': ([ann(CANCEL_BY), ann(CANCELED), FULL], + dict(conclusion='cancelled')), + 'a cancel while the job failed for memory': ([ann(CANCEL_BY), ann(EXIT_137), FULL], {}), + 'a cancel on a stalled machine at the time limit': ([ann(CANCEL_BY), ann(JOB_LIMIT), STALLED], + dict(conclusion='cancelled', took=900, + steps=ran('cancelled', took=880))), + 'a cancel after a lost runner': ([ann(CANCEL_BY), ann(LOST)], dict(steps=died())), + 'a cancel with no message': ([ann(CANCELED), FULL], dict(conclusion='cancelled')), + } + for label, (anns, kw) in cases.items(): + with self.subTest(label): + gh = one(anns, kw, conclusion='cancelled') + self.assertLeft(gh, *sweep(gh)[:2]) + def test_the_cancel_message_names_why(self): + gh = one([ann(CANCEL_BY), ann(EXIT_137), FULL]) + self.assertLeft(gh, *sweep(gh)[:2], "a person's cancel stands") -class NeverTwice(unittest.TestCase): + +class NeverTwice(Rerun): def test_a_second_failure_after_the_automatic_rerun_stays_red(self): - gh = one([ann(LOST)], attempt=2, by=BOT) + gh = lost(attempt=2, by=ACTIONS) code, out, text = sweep(gh) - self.assertEqual((code, gh.posts()), (0, []), out) - self.assertIn('a repeat red stays red', out) - self.assertIn('1 run(s) whose latest attempt was an automatic re-run', out) - self.assertIn('1 run(s) whose latest attempt was an automatic re-run', text) + self.assertLeft(gh, code, out, 'a repeat red stays red') + self.assertFalse([p for _, p in gh.calls if '/jobs' in p or 'pulls/' in p], 'it costs no further read') def test_a_person_rerunning_after_the_automatic_rerun_gets_no_second_one(self): - gh = one([ann(LOST)], attempt=3, by=OWNER) - gh.attempts[(1, 2)] = run(attempt=2, by=BOT) - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), [], out) - self.assertIn('attempt 2 was triggered by', out) - self.assertIn('never twice', out) + for bot_attempt in (1, 2): # the loop reads every earlier attempt, down to the first + with self.subTest(bot_attempt=bot_attempt): + gh = lost(attempt=3, by=OWNER) + gh.attempts[(1, bot_attempt)] = run(attempt=bot_attempt, by=ACTIONS) + code, out, _ = sweep(gh) + self.assertLeft(gh, code, out, f'attempt {bot_attempt} was triggered by') + self.assertIn('never twice', out) def test_a_person_rerun_without_an_automatic_one_may_get_it(self): - gh = one([ann(LOST)], attempt=2, by=OWNER) # attempt 1 by the owner too (the fake's default) - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + gh = lost(attempt=2, by=OWNER) # attempt 1 by the owner too (the fake's default) + self.assertReran(gh, sweep(gh)[1]) - def test_the_watcher_runs_again_on_the_run_it_reran(self): - gh = one([ann(LOST)]) + def test_the_sweep_runs_again_on_the_run_it_reran(self): + gh = lost() sweep(gh) self.assertEqual(len(gh.posts()), 1) code, out, _ = sweep(gh) # the re-run is queued: not completed, nothing to do - self.assertEqual((code, len(gh.posts())), (0, 1), out) + self.assertEqual((code, len(gh.posts()), len(gh.comments())), (0, 1, 1), out) gh.runs[1].update(status='completed', conclusion='failure') # and it failed again, the same way code, out, _ = sweep(gh) - self.assertEqual((code, len(gh.posts())), (0, 1), out) + self.assertEqual((code, len(gh.posts()), len(gh.comments())), (0, 1, 1), out) self.assertIn('a repeat red stays red', out) def test_the_request_sent_twice_github_refuses_the_second(self): - gh = one([ann(LOST)]) + gh = lost() first = copy.deepcopy(gh.runs[1]) sweep(gh) - # A second watcher (or the fleet sweep) read the run before the first re-ran it and asks again. - done, how = cr.rerun(gh, first) + # A second pass (or the fleet sweep) read the run before the first re-ran it and asks again. + done, how = cr.rerun(gh, first, lambda: self.fail('no second record')) self.assertFalse(done) self.assertIn('already re-run', how) self.assertEqual(len(gh.posts()), 1, 'the second request is not even sent: the run is read again first') + def test_a_run_started_again_after_it_was_listed_is_not_asked_for(self): + gh = lost() + gh.on_read = lambda r: r.update(status='in_progress') # same attempt, running again (a person's re-run) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (0, [], []), out) + self.assertIn('already re-run or running again', out) + def test_github_refusing_a_rerun_that_already_started_is_no_trouble(self): - gh = one([ann(LOST)]) - # Another watcher re-ran it between this pass's last read and its request. - def race(): - gh.runs[1].update(run_attempt=2, status='queued', triggering_actor={'login': BOT}) + gh = lost() + def race(): # another pass re-ran it between this pass's last read and its request + gh.runs[1].update(run_attempt=2, status='queued', triggering_actor={'login': ACTIONS}) return urllib.error.HTTPError('x', 403, 'This workflow is already running', {}, None) gh.post_fails = race - code, out, text = sweep(gh) + code, out, _ = sweep(gh) self.assertEqual(code, 0, out) self.assertIn('already re-run (GitHub refused a second request: HTTP 403)', out) - def test_a_refusal_with_the_run_unchanged_is_trouble_and_tried_again(self): - gh = one([ann(LOST)]) + def test_a_refusal_with_the_run_unchanged_is_trouble_and_tried_again_with_one_comment(self): + gh = lost() gh.post_fails = urllib.error.HTTPError('x', 500, 'Server Error', {}, None) code, out, text = sweep(gh) self.assertEqual(code, 1, out) self.assertIn('the next pass tries again', out) self.assertIn('not read', text) + self.assertEqual(len(gh.comments()), 1, 'the reason is on record before the request') code, out, _ = sweep(gh) # the next pass self.assertEqual(code, 0, out) - self.assertEqual(len(gh.posts()), 2) - self.assertEqual(gh.runs[1]['run_attempt'], 2) + self.assertEqual((len(gh.posts()), gh.runs[1]['run_attempt'], len(gh.comments())), (2, 2, 1)) - def test_the_watcher_stopped_half_way_the_request_never_reached_github(self): - gh = one([ann(LOST)]) + def test_stopped_half_way_the_request_never_reached_github(self): + gh = lost() gh.post_fails = urllib.error.URLError('timed out') code, out, _ = sweep(gh) self.assertEqual((code, gh.runs[1]['run_attempt']), (1, 1), out) code, out, _ = sweep(gh) # the next pass finishes it, once self.assertEqual((code, gh.runs[1]['run_attempt']), (0, 2), out) - self.assertEqual(len(gh.posts()), 2) sweep(gh) - self.assertEqual(len(gh.posts()), 2) + self.assertEqual((len(gh.posts()), len(gh.comments())), (2, 1)) - def test_the_watcher_stopped_half_way_the_rerun_was_asked_but_not_started(self): - gh = one([ann(LOST)]) - gh.lost_reply = True # GitHub made attempt 2; the watcher never heard back + def test_a_reply_lost_after_github_took_the_request_still_leaves_the_why(self): + gh = lost() + gh.lost_reply = True # GitHub made attempt 2; the sweep never heard back code, out, _ = sweep(gh) self.assertEqual(code, 1, out) + [comment] = gh.comments() + self.assertIn('', comment) + self.assertIn('lost', comment) + self.assertIn(gh.runs[1]['html_url'], comment) gh.lost_reply = False for _ in range(2): # queued, never started: the next passes leave it alone code, out, _ = sweep(gh) self.assertEqual((code, len(gh.posts())), (0, 1), out) gh.runs[1].update(status='completed', conclusion='cancelled') # it never ran (cancelled while queued) code, out, _ = sweep(gh) - self.assertEqual(len(gh.posts()), 1, out) + self.assertEqual((len(gh.posts()), len(gh.comments())), (1, 1), out) self.assertIn('a repeat red stays red', out) + self.assertIn('1 automatic re-run(s) asked in the last 24 h', out) + + def test_no_record_no_rerun(self): + gh = lost() + gh.comment_fails = True # the notice cannot be written: the re-run would be hidden, so none + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts()), (1, []), out) + gh.comment_fails = False + code, out, _ = sweep(gh) + self.assertEqual((code, len(gh.posts()), len(gh.comments()), len(gh.issues)), (0, 1, 1, 1), out) + + +class TheNotice(Rerun): + def test_every_rerun_is_a_comment_on_one_issue_opened_once(self): + jobs = {1: [job(11, steps=died()), job(13, FOLLOWER, steps=[])], 2: [job(21), job(23, FOLLOWER, steps=[])]} + gh = Fake([run(1), run(2, branch='feature', event='pull_request')], jobs, + {11: [ann(LOST)], 21: [ann(EXIT_137), FULL]}) + code, out, text = sweep(gh) + self.assertEqual((code, len(gh.posts()), len(gh.issues)), (0, 2, 1), out) + [issue] = gh.issues + self.assertEqual((issue['title'], issue['labels'], issue['user']['login']), (TITLE, CFG['notice']['labels'], + BOT)) + bodies = {re.search(r'run=([0-9]+)', c['body'])[1]: c['body'] for c in issue['comments']} + self.assertEqual(sorted(bodies), ['1', '2']) + self.assertIn('what=rerun', bodies['1']) + second = {'body': bodies['2']} + self.assertIn('box /fleet.slice/job-1.service: max 8192 MB high 6144 MB', second['body']) + self.assertIn('OOM kill', second['body']) + self.assertIn(f'issue #{issue["number"]}', text) + self.assertIn('2 automatic re-run(s) asked in the last 24 h', text) + # A later pass, a later run: the same issue. + gh.runs[3] = run(3, created=T0 - 60) + gh.jobs[3] = [job(31, steps=died())] + gh.ann[31] = [ann(LOST)] + sweep(gh) + self.assertEqual((len(gh.issues), len(issue['comments'])), (1, 3)) + + def test_only_an_open_issue_the_bot_opened_with_exactly_the_title_is_ours(self): + gh = lost() + gh.issues = [ + {'number': 3, 'title': TITLE, 'state': 'open', 'user': {'login': OWNER}, 'comments': []}, + {'number': 4, 'title': TITLE + ' (old)', 'state': 'open', 'user': {'login': BOT}, 'comments': []}, + {'number': 5, 'title': TITLE, 'state': 'closed', 'user': {'login': BOT}, 'comments': []}, + {'number': 6, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': [], + 'pull_request': {}}, + ] + sweep(gh) + [ours] = [i for i in gh.issues if i['comments']] + self.assertNotIn(ours['number'], (3, 4, 5, 6)) + self.assertEqual((ours['title'], ours['user']['login']), (TITLE, BOT)) + + def test_two_passes_opening_it_at_once_keep_the_lowest(self): + gh = lost() + real = gh.call + def call(path, method='GET', data=None): + got = real(path, method, data) + if method == 'POST' and path == 'issues' and len(gh.issues) == 1: + # A dispatched pass opened one too, just before this one. + gh.issues.insert(0, {'number': 10, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, + 'comments': []}) + return got + gh.call = call + sweep(gh) + lowest, mine = sorted(gh.issues, key=lambda i: i['number']) + self.assertEqual((lowest['number'], len(lowest['comments'])), (10, 1)) + self.assertEqual((mine['state'], mine['comments'][0]['body']), ('closed', 'Duplicate of #10.')) + + def test_the_24_hour_count_reads_every_comment_past_hundreds_of_reruns(self): + gh = Fake([]) + gh.issues = [{'number': 2, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': []}] + comments = gh.issues[0]['comments'] + for n in range(350): # one every 4 minutes: 350 in the last 24 h + comments.append({'id': n, 'user': {'login': BOT}, 'created_at': iso(T0 - 26 * 3600 + n * 240), + 'body': f''}) + comments.append({'id': 999, 'user': {'login': BOT}, 'created_at': iso(T0 - 60), + 'body': ''}) + comments.append({'id': 997, 'user': {'login': BOT}, 'created_at': iso(T0 - 30 * 3600), + 'updated_at': iso(T0 - 60), 'body': ''}) + comments.append({'id': 998, 'user': {'login': OWNER}, 'created_at': iso(T0 - 60), + 'body': ''}) + inside = sum(1 for c in comments[:350] if cr.stamp(c['created_at']) >= T0 - 24 * 3600) + self.assertGreater(inside, 300) + code, out, _ = sweep(gh) + self.assertEqual(code, 0, out) + self.assertIn(f'{inside} automatic re-run(s) asked in the last 24 h (issue #2)', out) + self.assertEqual(sum(1 for _, p in gh.calls if p.startswith('issues/2/comments')), 4, 'four pages, no more') + + def test_no_issue_yet_counts_zero_and_opens_nothing(self): + code, out, text = sweep(Fake([])) + self.assertEqual(code, 0, out) + self.assertIn('0 run(s) re-run by this pass; 0 automatic re-run(s) asked in the last 24 h (no tracking ' + 'issue yet)', out) + self.assertIn('### fork-ci re-run', text) + + def test_fleet_mode_reports_once_and_does_not_rerun(self): + gh = lost() + for _ in range(3): + code, out, text = sweep(gh, env={SKIP_VAR: SKIP_VALUE}) + self.assertEqual((code, gh.posts()), (0, []), out) + self.assertIn(f'not re-run ({SKIP_VAR}={SKIP_VALUE})', text) + self.assertEqual(len(gh.comments()), 1) + self.assertIn('what=report', gh.comments()[0]) + self.assertIn('re-run it by hand', gh.comments()[0]) + _, first, _ = sweep(lost(), env={SKIP_VAR: SKIP_VALUE}) + self.assertIn('::error title=fork-ci re-run::', first) + self.assertNotIn('::error', out, 'reported once: no repeated error note') + self.assertIn('reported already', out) + self.assertIn('0 automatic re-run(s)', out, 'a report is not a re-run') + gh = lost() + sweep(gh, env={SKIP_VAR: 'auto'}) + self.assertEqual(len(gh.posts()), 1) + + +class Paging(unittest.TestCase): + """GitHub.pages, the script's real paging code, over a fake call().""" + + class Pager: + pages = cr.GitHub.pages + def __init__(self, items, shift_at=None): + self.items, self.shift_at, self.paths = items, shift_at, [] -class Scope(unittest.TestCase): + def call(self, path): + self.paths.append(path) + n = int(re.search(r'[?&]page=([0-9]+)', path)[1]) + if n == self.shift_at: # two new items arrived: the next page starts two earlier + self.items = [{'id': -1}, {'id': -2}] + self.items + return {'list': self.items[(n - 1) * 100:n * 100]} + + def test_every_page_until_a_short_one_each_item_once(self): + p = self.Pager([{'id': i} for i in range(320)], shift_at=2) + got = p.pages('x?a=1', 'list') + self.assertEqual(sorted(i['id'] for i in got), list(range(320)), 'none lost, none twice') + self.assertEqual(p.paths, [f'x?a=1&per_page=100&page={n}' for n in (1, 2, 3, 4)]) + + def test_an_exact_multiple_reads_the_empty_page_after(self): + p = self.Pager([{'id': i} for i in range(200)]) + self.assertEqual(len(p.pages('x', 'list')), 200) + self.assertEqual(p.paths[-1], 'x?per_page=100&page=3') + + def test_more_than_most_pages_is_an_error_not_a_short_list(self): + with self.assertRaises(ValueError): + self.Pager([{'id': i} for i in range(300)]).pages('x', 'list', most=3) + + def test_a_reply_that_is_not_a_list_is_an_error(self): + p = self.Pager([]) + p.call = lambda path: {'list': {'id': 1}} + with self.assertRaises(ValueError): + p.pages('x', 'list') + + def test_the_sweep_reads_every_page_of_runs(self): + runs = [run(rid, conclusion='success') for rid in range(2, 352)] + [run(1)] # the failed one listed last + gh = Fake(runs, {1: [job(11, steps=died())]}, {11: [ann(LOST)]}) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts()), (0, ['actions/runs/1/rerun-failed-jobs']), out) + self.assertEqual(sum(1 for _, p in gh.calls if p.startswith('actions/workflows/')), 4) + + def test_a_listing_past_the_page_limit_reruns_nothing_and_says_so(self): + gh = Fake([run(rid, conclusion='success') for rid in range(1, 1002)]) + code, out, text = sweep(gh) + self.assertEqual((code, gh.posts()), (1, []), out) + self.assertIn('not read', text) + + +class Scope(Rerun): def test_untrusted_runs_are_left_alone(self): cases = {'a stranger': dict(actor='someone-else', by='someone-else'), + 'started by a stranger, re-run by the owner': dict(actor='someone-else', attempt=2), 'triggered by a stranger': dict(by='someone-else'), 'a pull request from a fork': dict(event='pull_request', branch='feature', head_repo='someone-else/t3code'), 'a push to another branch': dict(branch='feature'), + 'another repository': dict(repo='someone-else/t3code'), 'another event': dict(event='workflow_dispatch'), 'another workflow': dict(workflow='other.yml')} for label, kw in cases.items(): with self.subTest(label): - gh = one([ann(LOST)], **kw) + gh = lost(**kw) code, out, _ = sweep(gh) - self.assertEqual((code, gh.posts()), (0, []), out) - self.assertIn('not a trusted run', out) + self.assertLeft(gh, code, out, 'not a trusted run') self.assertFalse([p for _, p in gh.calls if '/jobs' in p], 'an untrusted run costs no job read') + def test_a_pull_request_someone_else_opened_is_left_alone(self): + gh = lost(event='pull_request', branch='feature') + gh.pulls[7]['user'] = {'login': 'someone-else'} + self.assertLeft(gh, *sweep(gh)[:2], 'a pull request opened by') + + def test_a_push_the_upstream_sync_made_is_left_alone(self): + SYNCED.add(SHA) + try: + gh = lost() + self.assertLeft(gh, *sweep(gh)[:2], 'upstream sync') + finally: + SYNCED.discard(SHA) + def test_an_own_pull_request_is_rerun(self): - gh = one([ann(LOST)], event='pull_request', branch='feature') - code, out, _ = sweep(gh) - self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + gh = lost(event='pull_request', branch='feature') + self.assertReran(gh, sweep(gh)[1]) def test_a_branch_that_moved_on_or_is_gone_is_not_rerun(self): for heads in ({BRANCH: NEWER}, {}): with self.subTest(heads=heads): - gh = one([ann(LOST)]) + gh = lost() gh.heads = heads - code, out, _ = sweep(gh) - self.assertEqual((code, gh.posts()), (0, []), out) - self.assertIn('has moved on or is gone', out) + self.assertLeft(gh, *sweep(gh)[:2], 'has moved on or is gone') def test_runs_that_passed_are_still_running_or_ended_long_ago_cost_nothing(self): gh = Fake([run(1, conclusion='success'), run(2, status='in_progress', conclusion=None), run(3, updated=T0 - CFG['catch_up_minutes'] * 60 - 1)], {}, {}) code, out, text = sweep(gh) self.assertEqual(code, 0, out) - self.assertEqual([p for _, p in gh.calls if not p.startswith('actions/workflows/')], []) + self.assertEqual([p for _, p in gh.calls if not p.startswith(('actions/workflows/', 'issues?'))], []) self.assertIn('0 run(s) re-run by this pass', text) - def test_an_empty_list_of_runs(self): - code, out, text = sweep(Fake([])) - self.assertEqual(code, 0, out) - self.assertIn('0 run(s) re-run by this pass; 0 run(s)', out) - self.assertIn('### fork-ci re-run', text) - def test_one_run_by_id_ignores_the_catch_up_window(self): - gh = one([ann(LOST)], updated=T0 - 5 * 3600) - code, out, _ = sweep(gh, run_id=1) - self.assertEqual(gh.posts(), ['actions/runs/1/rerun-failed-jobs'], out) + gh = lost(updated=T0 - 5 * 3600, created=T0 - 30 * 3600) + self.assertReran(gh, sweep(gh, run_id=1)[1]) self.assertFalse([p for _, p in gh.calls if p.startswith('actions/workflows/')]) - def test_fleet_mode_reports_and_does_not_rerun(self): - gh = one([ann(LOST)]) - code, out, text = sweep(gh, env={SKIP_VAR: SKIP_VALUE}) - self.assertEqual((code, gh.posts()), (0, []), out) - self.assertIn('::error title=fork-ci re-run::', out) - self.assertIn('re-run it by hand', out) - self.assertIn(f'not re-run ({SKIP_VAR}={SKIP_VALUE})', text) - gh = one([ann(LOST)]) - sweep(gh, env={SKIP_VAR: 'auto'}) - self.assertEqual(len(gh.posts()), 1) - def test_a_github_error_on_one_run_does_not_stop_the_others(self): - gh = Fake([run(1), run(2)], {2: [job(21)]}, {21: [ann(LOST)]}) - real = gh.pages - def pages(path, key, most=10): + gh = Fake([run(1), run(2)], {2: [job(21, steps=died())]}, {21: [ann(LOST)]}) + real = gh.call + def call(path, method='GET', data=None): if path.startswith('actions/runs/1/'): raise urllib.error.URLError('reset') - return real(path, key, most) - gh.pages = pages + return real(path, method, data) + gh.call = call code, out, _ = sweep(gh) self.assertEqual(code, 1, out) self.assertEqual(gh.posts(), ['actions/runs/2/rerun-failed-jobs']) def test_a_pipe_in_a_message_keeps_the_summary_table_whole(self): - gh = one([ann('The hosted runner: a | b lost communication with the server.'), CALM]) - _, _, text = sweep(gh) + gh = one([ann('The hosted runner: a | b lost communication with the server.')], + dict(steps=died(), runner='a | b')) + _, out, text = sweep(gh) self.assertIn('a \\| b', text) - row = [l for l in text.splitlines() if l.startswith('| 1 |')][0] + row = [line for line in text.splitlines() if line.startswith('| 1 |')][0] self.assertEqual(row.count(' | '), 2, row) class Config(unittest.TestCase): - def test_the_real_config_is_valid_and_names_the_rules_parts(self): - self.assertEqual(cr.check_config(CFG), []) + def test_the_real_config_is_valid(self): + self.assertEqual(cr.check_config({k: v for k, v in CFG.items() if k != 'fleet'}), []) r = subprocess.run([sys.executable, '-I', SCRIPT, 'check'], capture_output=True, text=True) self.assertEqual(r.returncode, 0, r.stderr) - def test_github_messages_match_their_sign_and_nothing_else_does(self): - lost, limit = CFG['signs']['lost'], CFG['signs']['limit'] - for m in (LOST, SHUTDOWN): - self.assertTrue(cr.matches(lost, [m]) and not cr.matches(limit, [m]), m) + def test_github_messages_match_their_sign_as_a_whole_and_nothing_else_does(self): + def says(kind, message): + return bool(cr.github_says(CFG, kind, [ann(message)])) + for m in (LOST, HOSTED_LOST, SHUTDOWN): + self.assertTrue(says('lost', m) and not says('limit', m), m) for m in (JOB_LIMIT, STEP_LIMIT): - self.assertTrue(cr.matches(limit, [m]) and not cr.matches(lost, [m]), m) - for m in (EXIT_1, EXIT_137, CANCELED, 'Error: the test timed out after 5000 ms'): - self.assertFalse(cr.matches(lost + limit, [m]), m) + self.assertTrue(says('limit', m) and not says('lost', m), m) + self.assertTrue(says('cancelled_by', CANCEL_BY)) + for m in (EXIT_1, EXIT_137, CANCELED, 'Error: the test timed out after 5000 ms', 'Error: ' + LOST, + JOB_LIMIT + ' Also more.', 'lost communication with the server', 'x ' + CANCEL_BY, + 'The job has exceeded the maximum execution time of 15m0s'): + self.assertFalse(any(says(k, m) for k in cr.GROUPS), m) def test_bad_configs_are_refused(self): + base = {k: v for k, v in CFG.items() if k != 'fleet'} + def bad(change): - cfg = copy.deepcopy(CFG) + cfg = copy.deepcopy(base) change(cfg) return cr.check_config(cfg) cases = { 'a bad pattern': lambda c: c['signs']['lost'].append('(['), - 'an unknown sign': lambda c: c['signs'].__setitem__('exit', ['137']), - 'no actors': lambda c: c.__setitem__('actors', []), - 'an event a fork can cause': lambda c: c['events'].append('workflow_run'), + 'a pattern that is part of a message': lambda c: c['signs']['lost'].append('lost communication'), + 'a limit pattern without its minutes': lambda c: c['signs']['limit'].append('The job ran too long'), + 'an unknown sign': lambda c: c['signs'].__setitem__('exit', ['The exit']), + 'a missing sign': lambda c: c['signs'].pop('cancelled_by'), + 'no trust file': lambda c: c.__setitem__('trust', '../x'), 'a zero threshold': lambda c: c['stalled'].__setitem__('full_share_percent', 0), 'a missing threshold': lambda c: c['memory_full'].pop('max_events'), 'a string threshold': lambda c: c['memory_full'].__setitem__('oom_kills', '1'), 'an unknown key': lambda c: c.__setitem__('retries', 2), + 'an old key': lambda c: c.__setitem__('actors', [OWNER]), 'no skip variable': lambda c: c['skip_when'].pop('variable'), 'no window': lambda c: c.__setitem__('catch_up_minutes', -1), + 'a listing shorter than the window': lambda c: c.__setitem__('list_hours', 0.5), + 'no hosted group': lambda c: c.__setitem__('hosted_runner_groups', []), + 'no end step': lambda c: c.__setitem__('probe_end_step', ''), + 'a notice by a person': lambda c: c['notice'].__setitem__('author', OWNER), + 'a notice without a title': lambda c: c['notice'].__setitem__('title', ''), } for label, change in cases.items(): with self.subTest(label): self.assertTrue(bad(change), label) with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: - json.dump(dict(CFG, retries=2), f) + json.dump(dict(base, retries=2), f) try: r = subprocess.run([sys.executable, '-I', SCRIPT, 'check', '--config', f.name], capture_output=True, text=True) @@ -498,11 +897,22 @@ def bad(change): finally: os.unlink(f.name) + def test_the_workflow_must_be_one_the_fleet_trusts(self): + with tempfile.TemporaryDirectory() as d: + shutil.copy(os.path.join(ROOT, '.github', CFG['trust']), d) + path = os.path.join(d, 'fork-ci-rerun.json') + with open(path, 'w', encoding='utf-8') as f: + json.dump(dict({k: v for k, v in CFG.items() if k != 'fleet'}, workflow='other.yml'), f) + with self.assertRaisesRegex(ValueError, 'not one the trust config trusts'): + cr.load(path) + def test_the_script_holds_no_fact_of_this_repository(self): with open(SCRIPT, encoding='utf-8') as f: - text = f.read() - for fact in [REPO, OWNER, CFG['workflow'], FOLLOWER] + CFG['signs']['lost'] + CFG['signs']['limit']: - self.assertFalse(fact in text.replace('fork-ci-rerun', ''), f'the script names {fact!r}') + text = f.read().replace('fork-ci-rerun', '') + facts = [REPO, OWNER, CFG['workflow'], FOLLOWER, END_STEP, TITLE, HOSTED, CFG['trust']] + facts += [p for ps in CFG['signs'].values() for p in ps] + for fact in facts: + self.assertFalse(fact in text, f'the script names {fact!r}') class ProbeToDecider(unittest.TestCase): @@ -510,7 +920,8 @@ class ProbeToDecider(unittest.TestCase): def test_a_probe_notice_round_trips(self): n = {'seconds': 300, 'host': {'full_share': 40.0, 'full_avg300': 1.0, 'oom_kill': 0}, - 'cgroups': [{'path': '/a%b\nc', 'oom_kill': 2, 'max': 0, 'high': 0, 'full_share': None}]} + 'cgroups': [{'path': '/a%b\nc', 'max_mb': 4096, 'oom_kill': 2, 'max': 0, 'high': 0, + 'full_share': 40.0}]} out = io.StringIO() body = json.dumps({'status': 'failure', **n}, separators=(',', ':'), sort_keys=True) print(f'::notice title={probe.NOTE_TITLE}::{probe.command(probe.NOTE_HEAD + body)}', file=out) @@ -519,37 +930,51 @@ def test_a_probe_notice_round_trips(self): self.assertEqual(title, f'title={cr.NOTE_TITLE}') self.assertNotIn('\n', data) message = data.replace('%0A', '\n').replace('%0D', '\r').replace('%25', '%') # what the runner does - got, why = cr.note_of([ann(message, 'notice', cr.NOTE_TITLE)]) + anns = [ann(EXIT_137), ann(message, 'notice', cr.NOTE_TITLE)] + got, why = cr.note_of(CFG, job(1), anns) self.assertIsNone(why) self.assertEqual(got['cgroups'][0]['path'], '/a%b\nc') - self.assertEqual(cr.job_sign(CFG, [ann(EXIT_137), ann(message, 'notice', cr.NOTE_TITLE)])[0], 'memory full') + self.assertEqual(cr.job_sign(CFG, job(1), anns)[0], 'memory full') self.assertEqual((probe.NOTE_TITLE, probe.NOTE_HEAD), (cr.NOTE_TITLE, cr.NOTE_HEAD)) class Wiring(unittest.TestCase): def setUp(self): + import yaml with open(os.path.join(ROOT, '.github/workflows/fork-ci-rerun.yml'), encoding='utf-8') as f: - self.text = f.read() + self.wf = yaml.safe_load(f) def test_the_workflow_runs_the_sweep_on_a_schedule_with_least_permissions(self): - import yaml - wf = yaml.safe_load(self.text) + wf = self.wf self.assertEqual(set(wf[True]), {'schedule', 'workflow_dispatch'}) self.assertEqual(wf['permissions'], {}) sweep_job = wf['jobs']['sweep'] - self.assertEqual(sweep_job['permissions'], {'actions': 'write', 'checks': 'read', 'contents': 'read'}) + self.assertEqual(sweep_job['permissions'], {'actions': 'write', 'checks': 'read', 'contents': 'read', + 'issues': 'write'}) self.assertFalse(wf['concurrency']['cancel-in-progress'], 'a pass is never cut off by the next') step = sweep_job['steps'][-1] self.assertIn('python3 -I .github/scripts/fork-ci-rerun.py sweep', step['run']) self.assertEqual(step['env']['CI_RUNNERS'], '${{ vars.CI_RUNNERS }}') self.assertEqual(SKIP_VAR, 'CI_RUNNERS') - checkout = sweep_job['steps'][0]['with']['sparse-checkout'] - self.assertIn('/.github/fork-ci-rerun.json', checkout) - self.assertIn('/.github/scripts/fork-ci-rerun.py', checkout) + + def test_the_checkout_has_every_file_the_sweep_reads(self): + checkout = self.wf['jobs']['sweep']['steps'][0] + # persist-credentials stays (fork-contracts.test.py, checkout): after it only this script runs, with -I. + self.assertEqual(len(self.wf['jobs']['sweep']['steps']), 2) + sparse = checkout['with']['sparse-checkout'].split() + for path in ('.github/fork-ci-rerun.json', '.github/scripts/fork-ci-rerun.py', f'.github/{CFG["trust"]}', + '.github/scripts/fork-fleet.py'): + self.assertIn(f'/{path}', sparse) + + def test_a_dispatch_is_never_replaced_by_a_scheduled_pass(self): + # GitHub keeps one pending run per concurrency group: a dispatch has a group of its own. + group = self.wf['concurrency']['group'] + self.assertEqual(group, "fork-ci-rerun-${{ github.event_name == 'workflow_dispatch' && github.run_id " + "|| 'schedule' }}") def test_the_sweep_is_offset_from_the_fleet_sweep(self): import yaml - ours = yaml.safe_load(self.text)[True]['schedule'][0]['cron'] + ours = self.wf[True]['schedule'][0]['cron'] with open(os.path.join(ROOT, '.github/workflows/fork-fleet.yml'), encoding='utf-8') as f: fleet = yaml.safe_load(f)[True]['schedule'][0]['cron'] self.assertEqual((ours.split()[0], fleet.split()[0]), ('5-55/10', '*/10')) diff --git a/.github/workflows/fork-ci-rerun.yml b/.github/workflows/fork-ci-rerun.yml index 5e91e8deb7148..6fda29b3b2492 100644 --- a/.github/workflows/fork-ci-rerun.yml +++ b/.github/workflows/fork-ci-rerun.yml @@ -1,24 +1,24 @@ name: Fork CI re-run # Fork only. A failed Fork tests run gets ONE automatic "re-run failed jobs" -# (GitHub's own), only when every failed job shows a known infrastructure sign: -# its runner lost communication, its box was full (OOM kills, memory.max hits, -# memory.high throttling while stalled), or it ran past its time limit while the -# machine was stalled. Never for a test failure (no sign: left red, fail closed) -# and never twice: the re-run is triggered by github-actions, and a run with an -# attempt by anyone but CONFIG's actors is left red. Every rule, and the numbers -# that decide it, are in .github/scripts/fork-ci-rerun.py and -# .github/fork-ci-rerun.json; each job's numbers come from its own -# fork-ci-pressure.py steps (fork-tests.yml). Each re-run is a warning -# annotation with its signs and numbers and a row in this job's summary, and -# every pass counts the automatic re-runs of the last 24 hours. -# On a schedule, not on workflow_run (fork-contracts.test.py refuses events a -# pull request from another fork can cause): nothing is kept between passes, -# each reads the runs of the last hour from GitHub, so a pass that stopped -# half-way is finished by the next. Offset from fork-fleet.yml's sweep, which -# re-runs a lost fleet runner's run too: whichever comes first re-runs it, the -# other finds an attempt by github-actions and leaves it. Runbook: -# .github/fork-ci-rerun.md. +# (GitHub's own), only when every failed job shows a known infrastructure sign +# that GitHub's own runner message AND the job's own step record agree on: its +# runner was lost, its box was full and stalled (from the job's own pressure +# note, never on a GitHub-hosted runner), or it ran past its time limit while +# the machine was stalled. Never for a test failure (no sign: left red, fail +# closed), never for a person's cancel, never for a run the fleet would not +# trust, and never twice: the re-run is triggered by github-actions, and a run +# with an attempt by anyone else is left red. Every rule, and the numbers that +# decide it, are in .github/scripts/fork-ci-rerun.py and +# .github/fork-ci-rerun.json. Each re-run is first a comment on one tracking +# issue (opened once; the run, its signs and numbers), then a warning +# annotation and a row in this job's summary; the 24 h count is read from that +# issue. On a schedule, not on workflow_run (fork-contracts.test.py refuses +# events a pull request from another fork can cause): nothing is kept between +# passes, each reads GitHub, so a pass that stopped half-way is finished by the +# next. Offset from fork-fleet.yml's sweep, which re-runs a lost fleet runner's +# run too: whichever comes first re-runs it, the other finds an attempt by +# github-actions and leaves it. Runbook: .github/fork-ci-rerun.md. on: schedule: - cron: "5-55/10 * * * *" @@ -31,8 +31,10 @@ on: permissions: {} +# Scheduled passes queue behind each other; a dispatch (one run id) has a group +# of its own, so a scheduled pass never replaces it while it waits. concurrency: - group: fork-ci-rerun + group: fork-ci-rerun-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'schedule' }} cancel-in-progress: false jobs: @@ -44,13 +46,16 @@ jobs: actions: write checks: read contents: read + issues: write steps: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: sparse-checkout: | /.github/fork-ci-rerun.json + /.github/fork-fleet.json /.github/scripts/fork-ci-rerun.py + /.github/scripts/fork-fleet.py sparse-checkout-cone-mode: false - name: Re-run a run that failed for infrastructure only, once diff --git a/.github/workflows/fork-tests.yml b/.github/workflows/fork-tests.yml index 95b4b9b3c78cf..edebaefc1fd79 100644 --- a/.github/workflows/fork-tests.yml +++ b/.github/workflows/fork-tests.yml @@ -749,6 +749,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start - name: Reject repository-owned PR assets @@ -807,6 +808,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} @@ -833,6 +835,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start # Runs in the background while Vite+ installs; the step before the build waits for it. @@ -975,6 +978,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} @@ -1009,6 +1013,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start # A shard with nothing to run says so in the summary and stops here. @@ -1117,6 +1122,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} @@ -1152,6 +1158,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start - name: Setup Vite+ @@ -1206,6 +1213,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} @@ -1243,6 +1251,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start - name: Detect affected jobs @@ -1366,6 +1375,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} @@ -1392,6 +1402,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start - name: Setup Rust @@ -1414,6 +1425,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} @@ -1473,6 +1485,7 @@ jobs: # Memory pressure from here on, for the one automatic re-run of an infrastructure failure # (fork-ci-rerun.yml reads the note the end step leaves; fork-ci-pressure.py never fails the job). - name: Note memory pressure at the start + timeout-minutes: 1 run: python3 -I .github/scripts/fork-ci-pressure.py start - name: Setup Vite+ @@ -1486,6 +1499,7 @@ jobs: # Last: the numbers since the start step in the log, and as a notice when the job did not succeed. - name: Note memory pressure at the end + timeout-minutes: 1 if: ${{ always() && hashFiles('.github/scripts/fork-ci-pressure.py') != '' }} env: JOB_STATUS: ${{ job.status }} From 0861b26e1febca63221b018d428519c9ae0976a4 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 03:20:05 +0000 Subject: [PATCH 4/4] fork CI re-run: time limits from the workflow file, quoted text made safe (#130) - A time-limit message counts only when its minutes are the job's (or the step's) own timeout-minutes in the workflow file at the run's commit, read through the contents API by a strict reader (checked against PyYAML on the real workflow). A failing step printing GitHub's exact "after 1 minutes" line is no longer re-run. - Text quoted into the public tracking issue is one line, with no @mention and no HTML; only a comment's first line is read as its marker. - Tests: a run re-run and red again between listing and fresh read is not asked for (pins the run_attempt re-check); a tracking issue that cannot be read ends the pass red with the reason and asks no re-run. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-rerun.json | 3 +- .github/fork-ci-rerun.md | 10 +- .github/scripts/fork-ci-rerun.py | 131 +++++++++++++++++++++++--- .github/scripts/fork-ci-rerun.test.py | 118 ++++++++++++++++++++++- 4 files changed, 244 insertions(+), 18 deletions(-) diff --git a/.github/fork-ci-rerun.json b/.github/fork-ci-rerun.json index 13c9550136003..4487aa09a5bd8 100644 --- a/.github/fork-ci-rerun.json +++ b/.github/fork-ci-rerun.json @@ -13,7 +13,8 @@ "limit_slack_seconds: a time-limit message counts only when the job (or the step) ran its minutes less this.", "signs: GitHub's own runner messages, each pattern a whole message (re.fullmatch), only on an annotation in the", " runner's form (failure, no title, .github), and only together with the job's step record (the script's rules).", - " lost: the runner went away. limit: the job or a step ran past its time limit; only with `stalled`.", + " lost: the runner went away. limit: the job or a step ran past its time limit; only with `stalled`, and only", + " when its minutes are that job's or step's own timeout-minutes in `workflow` at the run's commit.", " cancelled_by: a person cancelled it; the run stays red.", "stalled: the job's memory pressure note shows the machine or a box stalled: some-or-all-tasks-waiting (full) share", " of the job's time, or the 5-minute full average at its end.", diff --git a/.github/fork-ci-rerun.md b/.github/fork-ci-rerun.md index f1210197ecf6e..fe3a1c8a0971d 100644 --- a/.github/fork-ci-rerun.md +++ b/.github/fork-ci-rerun.md @@ -66,7 +66,11 @@ start sample, a reboot in between, a file the runner cannot see. `memory.max` hits or `memory.high` throttling. One event without the stall, or a machine-wide OOM kill, is no sign. - **limit while stalled**: GitHub says the job or a step ran past its time - limit; the job (or that step) really ran that long (less + limit; its minutes are that job's (or step's) own `timeout-minutes` in the + workflow file at the run's commit (read through the contents API; a step + can print GitHub's words with any number, so the message's number alone + never counts, and a job or step the reader cannot match stays red); the + job (or that step) really ran that long (less `limit_slack_seconds`); no other step failed; and the note shows the machine or a box stalled (full) at least `full_share_percent` of the job, or a 5-minute full average at least `full_avg300_percent` at the end. @@ -110,6 +114,7 @@ Nothing is kept between passes: each one reads GitHub (the runs and the issue). | A pass stops before its request reached GitHub | The comment is there, the run unchanged; the next pass (within `catch_up_minutes`) sends the request once and adds no second comment. | | A pass stops after GitHub took the request (reply lost) | The comment, written first, says why. The new attempt exists: the next passes see it queued or running, or later red with an attempt by `github-actions`. Never twice. | | The comment cannot be written | No re-run (it would be hidden): a warning, the pass ends red, the next pass tries both. | +| The tracking issue cannot be read | No re-run (its record cannot be checked) and no count: the pass ends red and says so; the next pass tries again. | | A dispatch with `run_id` while a scheduled pass waits | Its own concurrency group: it is never replaced by a scheduled pass. | | An old run, from before the probe steps | No note: only a lost runner can qualify. A time limit or a failure without the note is left red. | | A run that ended more than `catch_up_minutes` ago | Not looked at (except with `run_id`). Runs are listed by creation over `list_hours`, every page read; a listing past 10 pages ends the pass red, nothing re-run. | @@ -142,5 +147,8 @@ them); that is `fork-fleet.py`'s to change. - GitHub keeps at most 10 notices per step and 50 annotations per job: on a job with very many annotations the note can be dropped. The job then reads as "no note" and stays red (fail closed). +- Text quoted into the public tracking issue (annotation messages, note + paths) is made one line with no `@` mention and no HTML, and only a + comment's first line is read as its marker. - A job a step's own time limit failed counts as `limit` only with a stall; a slow test on a calm machine stays red. diff --git a/.github/scripts/fork-ci-rerun.py b/.github/scripts/fork-ci-rerun.py index cb78aac91c073..02d1e07df4b39 100644 --- a/.github/scripts/fork-ci-rerun.py +++ b/.github/scripts/fork-ci-rerun.py @@ -24,9 +24,10 @@ the job's own), and its own pressure note shows a box with a memory limit stalled (full) at least memory_full.box_full_share_percent of the job AND OOM kills, memory.max hits or memory.high throttling in it. A machine-wide OOM kill, or one event without the stall, is no sign; - limit GitHub says the job or a step ran past its time limit (signs.limit, with the minutes), the job (or - that step) really ran that long, no other step failed, and the note shows the machine or the box - stalled (CONFIG stalled). + limit GitHub says the job or a step ran past its time limit (signs.limit, with the minutes), those minutes + are that job's (or step's) own timeout-minutes in the workflow file at the run's commit (never the + message's word alone: a step can print any number), the job (or that step) really ran that long, no + other step failed, and the note shows the machine or the box stalled (CONFIG stalled). The note counts only when the probe's end step completed (so the job wrote its own) and is the job's only one. One failed job without a sign (a test failure, no note, a garbled note, two notes) leaves the run red: fail closed. GitHub's "canceled by" (signs.cancelled_by) on any failed job leaves it red: a person's cancel stands. @@ -40,7 +41,7 @@ runs again finds the bot's attempt (and the issue's marker) and does nothing. Runbook: .github/fork-ci-rerun.md. Standard library only. """ -import argparse, calendar, http.client, importlib.util, json, os, re, sys, time +import argparse, base64, calendar, http.client, importlib.util, json, os, re, sys, time import urllib.error, urllib.parse, urllib.request sys.dont_write_bytecode = True # loading fork-fleet.py must not leave __pycache__ in the tree @@ -312,9 +313,95 @@ def memory_full(cfg, job, note): return None -def limit_hit(cfg, job, annotations): - """GitHub's time-limit message, when the job's record agrees: the job (or the step it names) ran at least that - long and no other step failed. (message, None), or (None, why) when there is no such message.""" +def scalar(v): + """A one-line YAML scalar as the workflow writes it (plain, '...' or "..."), or None for anything else.""" + v = v.strip() + if len(v) >= 2 and v[0] == v[-1] == "'": + return v[1:-1].replace("''", "'") + if len(v) >= 2 and v[0] == v[-1] == '"': + return None if '\\' in v else v[1:-1] + v = re.sub(r'\s+#.*$', '', v) + return v if v and v[0] not in '|>&*!{["\'' else None + + +def workflow_limits(text): + """{job key: {'name', 'minutes', 'steps': [{'name', 'minutes'}]}} of a workflow file's timeout-minutes, read + strictly in the layout CONFIG workflow is written in (jobs at 2 spaces, their keys at 4, a step's `- ` at 6 and + its keys at 8). Whatever it cannot read gives no minutes, and a limit it cannot confirm leaves the run red.""" + jobs, job, section, cur, inside = {}, None, None, None, False + for line in text.splitlines(): + if not line.strip() or line.lstrip().startswith('#'): + continue + if not line.startswith(' '): + inside, job = line.rstrip() == 'jobs:', None + continue + m = re.fullmatch(r' ([A-Za-z_][A-Za-z0-9_-]*):\s*', line) if inside else None + if m: + job = jobs[m[1]] = {'name': None, 'minutes': None, 'steps': []} + section = cur = None + continue + m = re.fullmatch(r' ([A-Za-z_-]+):(?:\s+(.*))?', line) if job else None + if m: + section, cur = m[1], None + if section in ('name', 'timeout-minutes'): + job['name' if section == 'name' else 'minutes'] = scalar(m[2] or '') + continue + m = re.fullmatch(r' (- | )([A-Za-z_-]+):(?:\s+(.*))?', line) if job and section == 'steps' else None + if not m: + continue + if m[1] == '- ': + cur = {'name': None, 'minutes': None} + job['steps'].append(cur) + if cur is not None and m[2] in ('name', 'timeout-minutes'): + cur['name' if m[2] == 'name' else 'minutes'] = scalar(m[3] or '') + for j in jobs.values(): + for x in [j] + j['steps']: + x['minutes'] = int(x['minutes']) if re.fullmatch(r'[0-9]{1,5}', x['minutes'] or '') else None + return jobs + + +def shown(template, text, matrix=False): + """Is `text` the name GitHub shows for this name template (each ${{ }} any text; a job whose name has none may + carry its matrix values in parentheses)?""" + parts = re.split(r'\$\{\{.*?\}\}', template) + tail = r'(?: \(.*\))?' if matrix and len(parts) == 1 else '' + return bool(re.fullmatch('.*'.join(map(re.escape, parts)) + tail, text, re.S)) + + +def limit_of(workflow, job_name, step=None): + """The timeout-minutes the workflow gives the job shown as job_name (or its step named `step`), or None when it + gives none, or when the name matches no job or more than one, or names steps with different limits.""" + keys = [k for k, j in workflow.items() if shown(j['name'] or k, job_name, matrix=True)] + if len(keys) != 1: + return None + j = workflow[keys[0]] + if step is None: + return j['minutes'] + got = {s['minutes'] for s in j['steps'] if s['name'] is not None and shown(s['name'], step)} + return got.pop() if len(got) == 1 else None + + +def workflow_reader(cfg, gh, run): + """limits(job name, step) from CONFIG workflow's file at the run's own commit, read once and only when asked.""" + got = [] + + def limits(job_name, step=None): + if not got: + sha = str(run.get('head_sha') or '') + if not re.fullmatch(r'[0-9a-f]{40}', sha): + raise ValueError(f'run {run.get("id")}: no commit to read the workflow at') + reply = gh.call(f'contents/.github/workflows/{cfg["workflow"]}?ref={sha}') + if reply.get('encoding') != 'base64': + raise ValueError(f'{cfg["workflow"]} at {sha[:12]}: not readable through the contents API') + got.append(workflow_limits(base64.b64decode(reply['content']).decode('utf-8'))) + return limit_of(got[0], job_name, step) + return limits + + +def limit_hit(cfg, job, annotations, limits=None): + """GitHub's time-limit message, when the job's record agrees: its minutes are the job's (or the step's) + timeout-minutes in the workflow (limits), the job (or the step it names) ran at least that long and no other + step failed. (message, None), or (None, why) when there is no such message.""" why = None for a in annotations: if not from_runner(a): @@ -330,6 +417,11 @@ def limit_hit(cfg, job, annotations): if others: why = f'GitHub\'s time-limit text, but step {others[0]!r} failed' continue + own = limits(job.get('name') or '', step) if limits else None + if own != int(m['minutes']): + why = (f'GitHub\'s time-limit text says {m["minutes"]} min, but the workflow gives ' + f'{"step " + repr(step) if step is not None else "the job"} {own} min: not GitHub\'s own') + continue if step is not None: ran = [took(s) or 0 for s in job.get('steps') or [] if s.get('name') == step and s.get('conclusion') in ('failure', 'cancelled')] @@ -355,7 +447,7 @@ def numbers(note): return '; '.join(out) -def job_sign(cfg, job, annotations): +def job_sign(cfg, job, annotations, limits=None): """(sign, evidence) when a failed job shows an infrastructure sign, else (None, why not).""" cancel = github_says(cfg, 'cancelled_by', annotations) if cancel: @@ -379,7 +471,7 @@ def job_sign(cfg, job, annotations): full = memory_full(cfg, job, note) if note else None if full: return 'memory full', f'{full} ({numbers(note)})' - hit, not_hit = limit_hit(cfg, job, annotations) + hit, not_hit = limit_hit(cfg, job, annotations, limits) if hit and note: stall = stalled(cfg, note) if stall: @@ -427,9 +519,9 @@ def decide(cfg, gh, run, env): red = [j for j in jobs if j.get('conclusion') in RED and j.get('name') not in cfg['followers']] if not red: return 'leave', 'no failed job of its own (only followers failed, or nothing did)' - evidence = [] + evidence, limits = [], workflow_reader(cfg, gh, run) for j in red: - sign, why = job_sign(cfg, j, annotations_of(gh, j)) + sign, why = job_sign(cfg, j, annotations_of(gh, j), limits) if not sign: return 'leave', f'{j.get("name")} ended {j.get("conclusion")}: {why}' evidence.append(f'{j.get("name")} ({j.get("conclusion")}): {sign}: {why}') @@ -473,7 +565,8 @@ def load(self, since=None): def marks(self): out = {} for c in self.comments or []: - for m in MARK.finditer(str(c.get('body') or '')): + m = MARK.match(str(c.get('body') or '')) # only a comment's first line: quoted text is never one + if m: out.setdefault((int(m[1]), int(m[2]), m[3]), c.get('created_at')) return out @@ -520,8 +613,8 @@ def add(self, run, what, text): f'**Not re-run (fleet mode)**: [run {rid}]({url}) attempt {attempt} qualifies for its one automatic ' 're-run; re-run it by hand.') body = (f'\n{head}\n\n' - f'{run.get("event")} `{run.get("head_branch")}` `{str(run.get("head_sha"))[:12]}`\n\n' - + ''.join(f'- {part.strip()}\n' for part in text.split(' | ')))[:LONGEST] + f'{quoted(run.get("event"))} `{quoted(run.get("head_branch"))}` `{str(run.get("head_sha"))[:12]}`\n\n' + + ''.join(f'- {quoted(part.strip())}\n' for part in text.split(' | ')))[:LONGEST] if self.issue is None: self.issue = self._open() got = self.gh.call(f'issues/{self.issue}/comments', method='POST', data={'body': body}) @@ -530,6 +623,13 @@ def add(self, run, what, text): return self.issue +def quoted(text): + """Text from annotations and notes, safe in the public comment: one line, no @mention (a zero-width joiner after + each @), and no HTML (so never a marker line, MARK, of another run's record).""" + return (str(text).replace('\r', ' ').replace('\n', ' ').replace('@', '@\u200d').replace('&', '&') + .replace('<', '<').replace('>', '>')) + + def rerun(gh, run, record): """GitHub's own re-run of failed jobs (and the jobs after them), once: (True, how) when this pass asked for it. record() writes the durable notice first, so a reply lost after GitHub took the request still leaves the why.""" @@ -622,7 +722,8 @@ def record(): f'{cfg["window_hours"]} h ({notice.where()})') except TROUBLE as e: trouble += 1 - count = f'{asked} run(s) re-run by this pass; the tracking issue could not be read ({e.__class__.__name__})' + count = (f'{asked} run(s) re-run by this pass; the tracking issue could not be read for the ' + f'{cfg["window_hours"]} h count ({e.__class__.__name__}: {e}): red, the next pass tries again') say(f'fork-ci re-run: {count}', out) lines = ['### fork-ci re-run', '', count, ''] if rows: diff --git a/.github/scripts/fork-ci-rerun.test.py b/.github/scripts/fork-ci-rerun.test.py index 2eb8056ea8ce4..d0bc7bb47063c 100644 --- a/.github/scripts/fork-ci-rerun.test.py +++ b/.github/scripts/fork-ci-rerun.test.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Tests of fork-ci-rerun.py (the one automatic re-run of an infrastructure failure), with a fake GitHub that pages like the real one (the script's own GitHub.pages reads it), and of its wiring in fork-ci-rerun.yml. Run: python3 .""" -import copy, io, json, os, re, shutil, subprocess, sys, tempfile, time, unittest, urllib.error, urllib.parse +import base64, copy, io, json, os, re, shutil, subprocess, sys, tempfile, time, unittest, urllib.error, urllib.parse import importlib.util sys.dont_write_bytecode = True # importing the script must not leave __pycache__ in the tree @@ -49,6 +49,32 @@ CANCELED = 'The operation was canceled.' EXIT_1 = 'Process completed with exit code 1.' EXIT_137 = 'Process completed with exit code 137.' +# The workflow file at the run's commit, as the contents API hands it back: the limits GitHub's messages must match. +WORKFLOW = f"""name: Fork tests +on: push +jobs: + test: + name: Test + strategy: + matrix: + shard: [1, 2] + timeout-minutes: 15 + steps: + - name: Checkout + - name: Note memory pressure at the start + timeout-minutes: 1 + run: | + python3 x.py + - name: Test + timeout-minutes: 10 # the step's own + - name: '{END_STEP}' + timeout-minutes: 1 + lint: + name: Lint + timeout-minutes: 20 + steps: + - run: true +""" MEMORY_WORDS = 'OOM kill: memory.max hit, memory full, the box stalled (full) 90% of the job' @@ -143,6 +169,8 @@ def __init__(self, runs, jobs=None, annotations=None, heads=None, pulls=None): self.lost_reply = False # the re-run POST reaches GitHub, its reply never comes back self.comment_fails = False # a comment POST fails self.on_read = None # called with the run before GET actions/runs/N answers + self.workflow = WORKFLOW # the workflow file at every commit (None: not found) + self.comments_fail = False # reading an issue's comments fails def call(self, path, method='GET', data=None): self.calls.append((method, path)) @@ -191,6 +219,11 @@ def paged(items): return {} if method != 'GET': raise AssertionError(f'unexpected {method} {path}') + if base == f'contents/.github/workflows/{CFG["workflow"]}': + assert q['ref'] == [SHA], q + if self.workflow is None: + raise urllib.error.HTTPError(path, 404, 'Not Found', {}, None) + return {'encoding': 'base64', 'content': base64.b64encode(self.workflow.encode()).decode()} if base == f'actions/workflows/{CFG["workflow"]}/runs': since = cr.stamp(urllib.parse.unquote(q['created'][0]).lstrip('>=')) listed = sorted((r for r in self.runs.values() if cr.stamp(r['created_at']) >= since), @@ -228,6 +261,8 @@ def paged(items): return paged(listed) m = re.fullmatch(r'issues/(\d+)/comments', base) if m: + if self.comments_fail: + raise urllib.error.URLError('comments: reset') since = cr.stamp(q['since'][0]) # GitHub's `since` is by updated_at: an old comment edited lately is listed too. return paged([c for c in self.issue(int(m[1]))['comments'] @@ -366,6 +401,47 @@ def test_a_time_limit_message_needs_the_time_really_spent(self): gh = one([ann(message), STALLED], kw) self.assertLeft(gh, *sweep(gh)[:2]) + def test_a_time_limit_message_counts_only_with_the_workflows_own_minutes(self): + """A failing step can print GitHub's exact words with any number of minutes: "after 1 minutes" makes a step + that ran 10 minutes look 10 times over its limit. Only the workflow's own timeout-minutes counts.""" + cases = { + "a failing step printing GitHub's timeout line": ("The action 'Test' has timed out after 1 minutes.", + dict(steps=ran('failure', took=600))), + 'the job form with the wrong minutes': (JOB_LIMIT.replace('15 minutes', '1 minutes'), + dict(conclusion='cancelled', steps=ran('cancelled', took=880), + took=900)), + 'a step with no limit of its own': ("The action 'Checkout' has timed out after 1 minutes.", + dict(steps=[step('Checkout', 'failure', took=600)] + + ran('skipped')[1:])), + } + for label, (message, kw) in cases.items(): + with self.subTest(label): + gh = one([ann(message), note(cgroups=[box(full_share=50.0)])], kw) + self.assertLeft(gh, *sweep(gh)[:2], 'not GitHub\'s own') + with self.subTest('a job the workflow does not have'): + gh = one([ann(STEP_LIMIT), STALLED], dict(name='Gone (1)', steps=ran('failure', took=600))) + self.assertLeft(gh, *sweep(gh)[:2], 'the workflow gives step \'Test\' None min') + with self.subTest('the workflow at that commit cannot be read'): + gh = one([ann(STEP_LIMIT), STALLED], dict(steps=ran('failure', took=600))) + gh.workflow = None + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (1, [], []), out) + + def test_the_workflow_reader_reads_every_limit_of_the_real_workflow(self): + import yaml + with open(os.path.join(ROOT, '.github/workflows', CFG['workflow']), encoding='utf-8') as f: + text = f.read() + real, mine = yaml.safe_load(text)['jobs'], cr.workflow_limits(text) + self.assertEqual(sorted(mine), sorted(real)) + for key, j in real.items(): + self.assertEqual((mine[key]['name'], mine[key]['minutes']), (j.get('name'), j.get('timeout-minutes')), key) + self.assertEqual([(s['name'], s['minutes']) for s in mine[key]['steps']], + [(s.get('name'), s.get('timeout-minutes')) for s in j.get('steps') or []], key) + wf = cr.workflow_limits(text) + self.assertEqual(cr.limit_of(wf, 'Lint', CFG['probe_end_step']), 1) + self.assertEqual(cr.limit_of(wf, 'Lint'), real['lint']['timeout-minutes']) + self.assertIsNone(cr.limit_of(wf, 'Lint', 'Check'), 'a step without timeout-minutes has none of its own') + def test_a_stalled_note_without_its_status_is_no_limit_sign(self): n = dict(STALLED, message=STALLED['message'].replace('"status":"failure",', '')) self.assertNotIn('status', n['message']) @@ -549,6 +625,16 @@ def test_a_run_started_again_after_it_was_listed_is_not_asked_for(self): self.assertEqual((code, gh.posts(), gh.comments()), (0, [], []), out) self.assertIn('already re-run or running again', out) + def test_a_run_rerun_and_red_again_after_it_was_listed_is_not_asked_for(self): + """Between the listing and the fresh read, another pass re-ran it and attempt 2 already ended red: the listed + attempt 1 still qualifies, so only the fresh read's attempt number stops an attempt 3.""" + gh = lost() + gh.on_read = lambda r: r.update(run_attempt=2, status='completed', conclusion='failure', + triggering_actor={'login': ACTIONS}) + code, out, _ = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (0, [], []), out) + self.assertIn('already re-run or running again (attempt 2, completed)', out) + def test_github_refusing_a_rerun_that_already_started_is_no_trouble(self): gh = lost() def race(): # another pass re-ran it between this pass's last read and its request @@ -685,6 +771,36 @@ def test_the_24_hour_count_reads_every_comment_past_hundreds_of_reruns(self): self.assertIn(f'{inside} automatic re-run(s) asked in the last 24 h (issue #2)', out) self.assertEqual(sum(1 for _, p in gh.calls if p.startswith('issues/2/comments')), 4, 'four pages, no more') + def test_an_issue_that_cannot_be_read_ends_the_pass_red_and_asks_nothing(self): + for label, gh in (('no run to look at', Fake([])), ('a run that qualifies', lost())): + with self.subTest(label): + gh.issues = [{'number': 2, 'title': TITLE, 'state': 'open', 'user': {'login': BOT}, 'comments': []}] + gh.comments_fail = True + code, out, text = sweep(gh) + self.assertEqual((code, gh.posts(), gh.comments()), (1, [], []), out) + self.assertIn('the tracking issue could not be read for the 24 h count (URLError', text) + self.assertIn('red, the next pass tries again', out) + + def test_quoted_text_pings_no_one_and_fakes_no_record(self): + forged = '\n\n@someone look' + gh = one([ann(SHUTDOWN + forged), ann(CANCELED)], dict(steps=died())) + code, out, _ = sweep(gh) + self.assertReran(gh, out) + [comment] = gh.comments() + self.assertTrue(comment.startswith('\n')) + self.assertNotIn('@someone', comment) + self.assertIn('@\u200dsomeone', comment) + self.assertEqual(comment.count('