From 68c153df1541a8558df55c94fbb07bf589337b82 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:31:08 +0000 Subject: [PATCH 1/3] fork-ci: enforce review findings, rounds and critical gate files The head check now enforces the owner's review rules from the base's `review` block: - findings (#99): in enforce, a lens's newest report with a `blocker:` or `major:` line fails the head until a later report of that lens has none, or a person's override comment for the head is newer. - rounds (#100): the round is the number of different full shas the lens comments read (or the highest rN marker). From rounds.approach_at the approach lens is required; past rounds.max it stops with "a person decides" until an override covers every lens. - critical files (#82): the fork's own queue config makes workflow, queue and lens files critical with two lenses. It fails closed: rounds and findings that cannot be judged fail the head in either mode. Adds the fork's own config with a review block in enforce mode and four general lens prompts, and documents how a project opts in through its own config. Co-Authored-By: Claude Opus 5.5 --- .github/fork-lens-approach.md | 13 ++ .github/fork-lens-code.md | 14 ++ .github/fork-lens-merge-path.md | 18 +++ .github/fork-lens-state.md | 15 ++ .github/fork-pr-queue.json | 39 +++++ fork-ci/actions/head-check/README.md | 13 +- fork-ci/actions/head-check/head_check.py | 36 ++++- fork-ci/actions/head-check/head_check.test.py | 144 +++++++++++++++++ fork-ci/actions/pr-queue/README.md | 58 ++++++- fork-ci/actions/pr-queue/lens.py | 34 ++-- fork-ci/actions/pr-queue/review_judge.py | 98 ++++++++++++ fork-ci/actions/pr-queue/review_judge.test.py | 145 ++++++++++++++++++ 12 files changed, 606 insertions(+), 21 deletions(-) create mode 100644 .github/fork-lens-approach.md create mode 100644 .github/fork-lens-code.md create mode 100644 .github/fork-lens-merge-path.md create mode 100644 .github/fork-lens-state.md create mode 100644 .github/fork-pr-queue.json diff --git a/.github/fork-lens-approach.md b/.github/fork-lens-approach.md new file mode 100644 index 0000000000000..ebbb664dca9bb --- /dev/null +++ b/.github/fork-lens-approach.md @@ -0,0 +1,13 @@ +# Lens: approach + +This pull request has reached its third review round or later. Do not look for +more small findings. Ask whether the whole approach is right: + +- Does it solve the problem the issue names, at the root? +- Is there a simpler way, or an existing mechanism it should build on? +- Will more rounds converge, or should it be split, restarted or closed? + +Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections +"Lens comments" and "Findings and rounds"): the first line is the lens header, +then one line per finding that starts `blocker:`, `major:` or `minor:`. +Write `major: none` when there is none. diff --git a/.github/fork-lens-code.md b/.github/fork-lens-code.md new file mode 100644 index 0000000000000..4e7fa12580eb9 --- /dev/null +++ b/.github/fork-lens-code.md @@ -0,0 +1,14 @@ +# Lens: code + +You review one pull request of this repository: is it correct, tested and as +simple as it can be? Read the diff and the files it touches; run its tests. + +- Correctness first: wrong results, crashes, missed cases. +- Ask: what happens when this runs again, stops half-way, or meets existing, + old or empty state? +- Tests: do they cover the change, and do they fail on the old code? + +Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections +"Lens comments" and "Findings and rounds"): the first line is the lens header, +then one line per finding that starts `blocker:`, `major:` or `minor:`. +Write `major: none` when there is none. diff --git a/.github/fork-lens-merge-path.md b/.github/fork-lens-merge-path.md new file mode 100644 index 0000000000000..2a02ca3c5a394 --- /dev/null +++ b/.github/fork-lens-merge-path.md @@ -0,0 +1,18 @@ +# Lens: merge path and security + +You review one pull request of this repository from one angle: can it let a +change merge that should not, or leak or weaken anything? Read the diff and +the files it touches; run its tests. + +- Every gate, check, ruleset and workflow it touches: can a head pass that no + review read, or a check pass without running? +- Fail closed: an unknown, unreadable or empty answer must block, never pass. +- Tokens and secrets: none printed, stored or handed to code the pull request + controls. +- This repository is public: nothing personal (names, addresses, paths of a + person's machine, tokens) in files, commits or comments. + +Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections +"Lens comments" and "Findings and rounds"): the first line is the lens header, +then one line per finding that starts `blocker:`, `major:` or `minor:`. +Write `major: none` when there is none. diff --git a/.github/fork-lens-state.md b/.github/fork-lens-state.md new file mode 100644 index 0000000000000..157fad02f6a05 --- /dev/null +++ b/.github/fork-lens-state.md @@ -0,0 +1,15 @@ +# Lens: state and tests + +You review one pull request of this repository from one angle: does it hold +when it meets real state, and do its tests prove it? Read the diff and the +files it touches; run its tests. + +- Ask every time: what happens when this runs again, stops half-way, or meets + existing, old or empty state? +- Each fix line should turn a test red when it is removed. Spot-check some. +- New tests should fail on the old code. Check one. + +Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections +"Lens comments" and "Findings and rounds"): the first line is the lens header, +then one line per finding that starts `blocker:`, `major:` or `minor:`. +Write `major: none` when there is none. diff --git a/.github/fork-pr-queue.json b/.github/fork-pr-queue.json new file mode 100644 index 0000000000000..65e4da643face --- /dev/null +++ b/.github/fork-pr-queue.json @@ -0,0 +1,39 @@ +{ + "required_checks": [ + "fork-ci-result", + "No database schema changes", + "Outside pull request", + "head_check" + ], + "merge_method": "squash", + "bot_logins": [], + "lens_author_associations": ["OWNER"], + "override": { "prefix": "override for head", "author_association": "OWNER" }, + "ready_command": "gh pr merge {pr} --auto --squash", + "push_command": "git push", + "lens_format_doc": "fork-ci/actions/pr-queue/README.md, section Lens comments", + "head_check": "head_check", + "review": { + "version": 1, + "mode": "enforce", + "rules": [ + { + "name": "gate", + "weight": "critical", + "paths": [".github/**", "fork-ci/**"], + "lenses": ["merge-path", "state"] + }, + { "name": "code", "weight": "normal", "paths": ["**"], "lenses": ["code"] } + ], + "lenses": { + "merge-path": { "prompt": ".github/fork-lens-merge-path.md" }, + "state": { "prompt": ".github/fork-lens-state.md" }, + "code": { "prompt": ".github/fork-lens-code.md" }, + "approach": { "prompt": ".github/fork-lens-approach.md" } + }, + "min_lenses": { "critical": 2, "normal": 1, "light": 0 }, + "rounds": { "approach_at": 3, "max": 4, "approach_lens": "approach" }, + "findings": true, + "state_line": false + } +} diff --git a/fork-ci/actions/head-check/README.md b/fork-ci/actions/head-check/README.md index 1e4195b5beee3..6b7d338f684b5 100644 --- a/fork-ci/actions/head-check/README.md +++ b/fork-ci/actions/head-check/README.md @@ -50,11 +50,22 @@ All of them are the queue's, imported from `../pr-queue`, never copied: summary only adds a row "the path rules would have said: ...". A change the rules cannot judge (no merge base, a failed diff) refuses, in either mode. Without the block, nothing of this runs. +- **Findings and rounds** (`review_judge.judge_rounds`, the pr-queue README, + "Findings and rounds"): with a `review` block in `enforce`, a lens whose + newest report has a `blocker:` or `major:` line fails the head until a later + round of that lens clears it or a person's override covers it + (`findings: true`); from round `rounds.approach_at` the approach lens is + required; a round past `rounds.max` fails the head until a person's override + covers every lens (no more rounds: the person overrides or closes). In + `shadow` the summary only adds a row "they would have said: ...". Comments + that cannot be judged fail the head, in either mode. It passes when: - every lens read the head, or a newer override vouches for it, and every - required lens (`required-lenses`) is among them; + required lens (`required-lenses`, and in `enforce` the path rules' and the + approach lens) is among them, with no open blocker or major and no round past + the cap (`enforce` only); - the pull request has no lens comments and no lens is required: there is nothing to compare (the queue then vouches only for the head it found); - it is a draft and `pass-drafts` is `true`. diff --git a/fork-ci/actions/head-check/head_check.py b/fork-ci/actions/head-check/head_check.py index 4da69ceba5a0b..3a0fa7d6eb731 100644 --- a/fork-ci/actions/head-check/head_check.py +++ b/fork-ci/actions/head-check/head_check.py @@ -19,7 +19,11 @@ The config's optional `review` block (../pr-queue/README.md, "Review rules") names lenses by changed path: the paths are `git diff --name-status -M` from the merge base to the head (changed_paths), judged by review_judge.judge. In `enforce` its lenses are required too; in `shadow` the summary only says what they would have been. A change it cannot -judge (no merge base, a failed diff) is refused, never passed. Without the block nothing of this runs. +judge (no merge base, a failed diff) is refused, never passed. The same block judges the review itself +(review_judge.judge_rounds, review_rounds): in `enforce` a lens's newest report with a `blocker:` or `major:` line that +no later round or override cleared fails the head (`findings: true`), the approach lens is required from round +`rounds.approach_at`, and a round past `rounds.max` fails the head until a person's override covers every lens; in +`shadow` the summary only says so. Without the block nothing of this runs. Pass: every lens read the head (or a newer override vouches for it), and every required lens is there; a pull request with no lens comments and no required lens (nothing to compare); a draft, with --pass-drafts only. Fail, @@ -344,12 +348,42 @@ def decide(repo, n, config_path, checkout, required, pass_drafts, gh, sleep, out return 1, False, f'could not read the pull request from GitHub after {ATTEMPTS} tries: {why}', [] review = lens.Review(cfg.bot_logins, cfg.override_prefix, cfg.override_association, cfg.lens_author_associations, log=lambda text: out(escape(text))) + rounds = None + if paths is not None and not (pr['isDraft'] and pass_drafts): + rounds, why = review_rounds(cfg.review, review, pr) + if rounds is None: # fail closed, in shadow as in enforce, as for the path rules + return 1, False, f'the review rounds and findings cannot be judged, so the head is not passed: {why}', [row] + if enforced and rounds.approach: + required.setdefault(lens.lens_key(cfg.review.rounds.approach_lens), cfg.review.rounds.approach_lens) ok, reason, rows = verdict(pr, review, checkout, required, pass_drafts, enforced) + if rounds is not None: + mode = cfg.review.mode + rows = [(f'findings and rounds ({mode})', not (enforced and rounds.problems), + ('' if enforced else 'they would have said: ') + rounds.text(cfg.review))] + rows + out(f'findings and rounds ({mode}): {escape(rows[0][2])}') + if enforced and rounds.problems: + reason = '; '.join(rounds.problems) + ('' if ok else f'; and: {reason}') + ok = False if paths is not None: rows = [row] + rows return (0 if ok else 1), ok, reason, rows +def review_rounds(rules, review, pr): + """(review_judge.Rounds, None) of the pull request's lens comments under the base's `review` rules, or (None, why). + Findings come from each lens's newest report that counts as a read; the round from every lens header that counts. + Any error is a why, never a pass.""" + try: + entries, history = review.lens_scan(pr) + reads = review.lens_reads(pr) + newest = {k: (entries[k][3].get('body') or '') if sha and not override else None + for k, (sha, _, override) in reads.items()} + overridden = {k for k, (_, _, override) in reads.items() if override} + return review_judge.judge_rounds(rules, history, newest, overridden), None + except Exception as e: # noqa: BLE001 - unknown never passes + return None, f'{type(e).__name__}: {review_judge.shown(str(e))[:300]}' + + def argument_problems(repo, n, required_text): """(required, problems) for the arguments both modes take.""" problems = [] diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index 413a070378368..f2d31225d6d78 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -1273,6 +1273,150 @@ def test_a_merge_tree_that_takes_too_long_is_refused(self): # #413 item 5: a t self.assertEqual(self.check(base, head, EVERY_LENS)[0], 0) # the same merge, given its time +# --- findings and rounds (#99, #100): the lens comments' blocker and major lines and how many rounds they read ----- +OLD = ['d4' * 20, 'e5' * 20, 'f6' * 20, '07' * 20] # heads earlier rounds read (gone from the branch now) + + +class FindingsAndRounds(unittest.TestCase): + """The base's `review` block (the shared vectors' rules: findings true, approach lens from round 3, cap 4) judges + the lens comments' findings and rounds. Each test changes one normal path, so the path rules need the lens code.""" + BARE = False + STORE = PathRules.STORE + setUp, base, head, check = PathRules.setUp, PathRules.base, PathRules.head, PathRules.check + + def said(self, name, sha, *lines, **kw): + return self.p.post(f'{LENS}: {name} · reviewer: x · read at: {sha}\n' + '\n'.join(lines), **kw) + + def pair(self, mode='enforce'): + b = self.base(mode) + return b, self.head(b, mode, **{'src/app.py': 'a\n'}) + + def test_a_major_in_the_newest_report_fails_the_head(self): + base, head = self.pair() + code, out = self.check(base, head, comments=[self.said('code', head, 'major: two writers race')]) + self.assertEqual(code, 1, out) + self.assertIn("lens code reported a major that no later round cleared (issue #99): 'major: two writers race'", + out) + self.assertIn('findings and rounds (enforce): round 1 of 4; lens code reported a major', out) + + def test_shadow_only_says_what_it_would_have_said(self): + base, head = self.pair('shadow') + code, out = self.check(base, head, comments=[self.said('code', head, '**Blocker:** data lost on restart')]) + self.assertEqual(code, 0, out) + self.assertIn('findings and rounds (shadow): they would have said: round 1 of 4; lens code reported a blocker', + out) + + def test_a_later_round_with_no_finding_clears_it(self): # runs again on the same pull request after a fix + base, head = self.pair() + old = self.said('code', OLD[0], 'major: two writers race') + code, out = self.check(base, head, comments=[old, self.said('code r2', head, 'major: none')]) + self.assertEqual(code, 0, out) + self.assertIn('findings and rounds (enforce): round 2 of 4; no blocker or major open', out) + + def test_a_round_stopped_half_way_keeps_the_old_finding(self): # the fix was pushed, the lens never read it + base, head = self.pair() + code, out = self.check(base, head, comments=[self.said('code', OLD[0], 'blocker: x')]) + self.assertEqual(code, 1, out) + self.assertIn('lens code reported a blocker', out) + + def test_a_deleted_newer_report_brings_the_old_finding_back(self): # deleted later: GitHub no longer lists it + base, head = self.pair() + first, second = self.said('code', head, 'blocker: x'), self.said('code r2', head, 'blocker: none') + self.assertEqual(self.check(base, head, comments=[first, second])[0], 0) + code, out = self.check(base, head, comments=[first]) + self.assertEqual(code, 1, out) + self.assertIn("lens code reported a blocker that no later round cleared (issue #99): 'blocker: x'", out) + + def test_an_edited_report_reads_nothing_and_still_counts_as_a_round(self): # edited later + base, head = self.pair() + comments = [self.said('code', OLD[0], edited=True), self.said('code', OLD[1], hidden=True), + self.said('code', head, 'major: fixed later', edited=True)] + code, out = self.check(base, head, comments=comments) + self.assertEqual(code, 1, out) + self.assertIn('round 3 of 4', out) # edited and hidden headers still count as rounds + self.assertNotIn('reported a major', out) # it reads nothing: the edit itself fails it + self.assertIn('edited', out) + + def test_the_approach_lens_is_required_from_round_three(self): + base, head = self.pair() + rounds = [self.said('code', OLD[0]), self.said('code', OLD[1])] + self.assertEqual(self.check(base, head, comments=rounds[:1] + [self.said('code', head)])[0], 0) + code, out = self.check(base, head, comments=rounds + [self.said('code', head)]) + self.assertEqual(code, 1, out) + self.assertIn('required lens without a read: approach', out) + code, out = self.check(base, head, comments=rounds + [self.said('code', head), self.said('approach', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + + def test_past_the_cap_it_stops_and_only_an_override_passes(self): + base, head = self.pair() + five = [self.said('code', sha) for sha in OLD] + [self.said('code', head), self.said('approach', head)] + code, out = self.check(base, head, comments=five) + self.assertEqual(code, 1, out) + self.assertIn('review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a person decides', out) + code, out = self.check(base, head, comments=five + [self.p.override(head)]) + self.assertEqual(code, 0, out) + stranger = self.p.override(head, assoc='CONTRIBUTOR', login='someone') # not the owner's: counts for nothing + self.assertEqual(self.check(base, head, comments=five + [stranger])[0], 1) + + def test_an_override_clears_a_wrong_finding(self): # #99: a person's override comment + base, head = self.pair() + report = self.said('code', head, 'blocker: wrong, the lock is there') + self.assertEqual(self.check(base, head, comments=[report])[0], 1) + self.assertEqual(self.check(base, head, comments=[report, self.p.override(head)])[0], 0) + override = self.p.override(head) + later = self.said('code r2', head, 'blocker: wrong again') # a report newer than the override counts again + self.assertEqual(self.check(base, head, comments=[report, override, later])[0], 1) + + def test_a_strangers_finding_neither_blocks_nor_counts_a_round(self): # anyone can comment on a public repo + base, head = self.pair() + noise = [self.said('code', sha, 'blocker: x', assoc='NONE', login='someone') for sha in OLD] + code, out = self.check(base, head, comments=noise + [self.said('code', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 1 of 4', out) + + def test_no_lens_comment_yet_is_round_zero(self): # empty state: the path rules' lens is still required + base, head = self.pair() + code, out = self.check(base, head, comments=[]) + self.assertEqual(code, 1, out) + self.assertIn('round 0 of 4; no blocker or major open', out) + self.assertIn('required lens without a read: code', out) + + def test_the_same_comments_give_the_same_verdict(self): # runs again + base, head = self.pair() + comments = [self.said('code', OLD[0], 'major: x'), self.said('code', head, 'major: y')] + self.assertEqual(self.check(base, head, comments=comments), self.check(base, head, comments=comments)) + + def test_no_review_block_judges_no_finding(self): # old state: a base config from before the block + base = self.base(mode=None) + head = self.head(base, mode=None, **{'src/app.py': 'a\n'}) + code, out = self.check(base, head, comments=[self.said('code', head, 'blocker: x')]) + self.assertEqual(code, 0, out) + self.assertNotIn('findings and rounds', out) + + def test_findings_false_judges_no_finding_but_still_counts_rounds(self): + b = self.w.commit(base_files(cfg_raw=review_cfg(findings=False), **{**PROMPTS, **self.STORE}), [self.m0]) + head = self.w.commit(base_files(cfg_raw=review_cfg(findings=False), + **{**PROMPTS, **self.STORE, 'src/app.py': 'a\n'}), [b]) + code, out = self.check(b, head, comments=[self.said('code', head, 'blocker: x')]) + self.assertEqual(code, 0, out) + self.assertIn('findings are not judged', out) + + def test_a_judge_that_fails_is_a_failed_head_in_either_mode(self): # fail closed: never a silent pass + for mode in ('enforce', 'shadow'): + with self.subTest(mode): + base, head = self.pair(mode) + with unittest.mock.patch.object(H.review_judge, 'judge_rounds', side_effect=ValueError('boom')): + code, out = self.check(base, head, comments=[self.said('code', head)]) + self.assertEqual(code, 1, out) + self.assertIn('the review rounds and findings cannot be judged, so the head is not passed: ' + 'ValueError: boom', out) + + +class FindingsAndRoundsBare(FindingsAndRounds): + BARE = True + + class GitEnvironment(unittest.TestCase): # #442 item 2: pr_queue.GIT_RUN """The gate's git calls get no program or transport of the job's environment where none can be needed: its local calls reach no remote, and its fetch from GitHub (https) needs git's own helpers only. The checkout mode fetches diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index d105dfa056d47..b586faecd92e4 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -61,11 +61,12 @@ does not match itself as a pattern, and the queue then refuses to start. The optional `review` block of the config says which changed paths need which lens reads and caps the review rounds. The head gate (`../head-check`) judges -the required lenses by changed path with it (below, "How the gate judges"); the -round cap, the approach lens and the `findings` / `state_line` lines are not -enforced yet. `review.recommended.json` (next to this file) is a complete -example with made-up paths; a test parses it. Real paths, lens names and prompt -files belong in the project's own config. +the required lenses by changed path with it (below, "How the gate judges"), and +the findings, the approach lens and the round cap from the lens comments +(below, "Findings and rounds"). The `state_line` key is parsed but not enforced +yet. `review.recommended.json` (next to this file) is a complete example with +made-up paths; a test parses it. Real paths, lens names and prompt files belong +in the project's own config (below, "Turning it on"). Every key is required and an unknown key is an error, as in the rest of the config. A wrong block makes the whole config invalid, naming every problem at @@ -80,7 +81,7 @@ once. | `min_lenses` | `{"critical", "normal", "light"}`: the fewest different lenses a rule of that weight must list, 0-10, never falling with weight. | | `rounds` | `{"approach_at", "max", "approach_lens"}`: from round `approach_at` the approach lens is required too; after round `max` a person must write an override comment or close the pull request. `max` is not below `approach_at`. | | `findings` | `true` or `false`: lens comments carry `blocker:`, `major:` and `minor:` lines, and a blocker or major at the head fails the gate. | -| `state_line` | `true` or `false`: a required lens comment needs a `state:` line to count. | +| `state_line` | `true` or `false`: a required lens comment needs a `state:` line to count. Not enforced yet. | The config is refused when: @@ -188,6 +189,51 @@ It judges the merge with the base of its run: two pull requests that each pass alone can still merge into a bad config unless the ruleset requires branches to be up to date, which the queue needs anyway. +**Findings and rounds.** The gate also judges the review itself, from the lens +comments it reads anyway (section "Lens comments"; `review_judge.judge_rounds`). +Owner decisions: issue #99 (findings), issue #100 (rounds). + +- **Findings** (`findings: true`): a lens report writes each finding on a line + of its own that starts `blocker:`, `major:` or `minor:` (any case; Markdown + list, quote and emphasis marks do not count; `Majors:`, `major 2:` and + `major (state):` are the same). A lens whose newest report that counts as a + read has a `blocker:` or `major:` line fails the head. Only an explicit + nothing after the colon is no finding (`major: none`, `nothing`, `no`, `0`, + `n/a`, `-`); any other text is one, also an empty one. Every line counts, in + code blocks and quotes too, so a report names an earlier finding it saw fixed + without that form (`fixed: the major of round 1 ...`). The ways out: fix it, + and the lens posts a new report (a later round: its newest comment counts) + without a blocker or major line; or a person posts an override comment for + the head (config `override`), which covers every lens whose report is older. + `minor:` lines never block. +- **Rounds:** each different full sha that the counted lens headers name is a + fix-and-review round, and a round marker in a lens name (`code r3`) counts as + that round, whichever is higher. Hidden and edited lens comments count too + (hiding old rounds does not lower the count); a deleted one does not, so keep + the round marker in the names. From round `rounds.approach_at` the lens + `rounds.approach_lens` is required as well. Round `rounds.max` is the last: + a round past it fails the head, whatever the lenses said, with "no more + rounds; a person decides", until a person's override comment newer than every + lens comment covers the head; else the pull request is closed. At round + `rounds.max` a blocker or major says the same. +- In `shadow` the summary adds a row "findings and rounds (shadow): they would + have said: ..." and nothing fails. Lens comments that cannot be judged fail + the head in both modes. Without the `review` key nothing of this runs. + +**Workflow, queue and lens files are critical** (owner decision, issue #82). +The built-in protected paths (above) make every change to `.github/workflows/`, +to the config itself and to a lens prompt need the critical lenses: the extra +review. A project whose gate or queue code lives in its own repository (the +fork's `fork-ci/`) adds a critical rule for those paths. + +**Turning it on.** A project opts in through its own config, never through +the fork: add the `review` block to the queue config its head gate reads +(`config` input), with its real paths, lens names and prompt files, in +`shadow` first. Read a few gate summaries, fix every config warning, then set +`mode` to `enforce` and dispatch the backfill (head-check README). The fork's +own config is [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), +in `enforce`. + **The empty input.** The gate refuses an empty `required-lenses` input (a head with no lens comment would pass), except when the config at the base tip has a `review` block in `enforce`: then the path rules name the lenses. A name that diff --git a/fork-ci/actions/pr-queue/lens.py b/fork-ci/actions/pr-queue/lens.py index f4c8671fea071..1a1fd5200b69b 100644 --- a/fork-ci/actions/pr-queue/lens.py +++ b/fork-ci/actions/pr-queue/lens.py @@ -437,16 +437,23 @@ def lens_posts(self, pr): return [(when, i, post) for i, (when, post) in enumerate(posts) if not self.is_bot(post)] def lens_entries(self, pr): - """{lens key: ((when, index), sha or None, why or None)}: each lens's NEWEST header, from the posts that count - (counts). A comment that looks like a lens report but is not one valid read (lens_problem) is an entry of its - own, keyed by the comment, with no sha: a correct new post of that lens does not replace it, and hiding it - does not clear it. It reads nothing until it is deleted, its first line is edited to `not a lens: ...` with no - lens header line left in it, or a newer override covers it (NO_HEADER_WAY_OUT). A lens comment edited after - posting (includesCreatedEdit not false: GitHub records that it changed, not what) or hidden (minimized) reads - nothing until that lens posts a new comment; so does a review whose flags were not read (FLAGS_UNREAD). A - lens-looking post that does not count is logged (log only, never a comment), so a wrong lens_author_associations - shows in the run.""" - found = {} + """{lens key: ((when, index), sha or None, why or None)}: lens_scan's entries without their posts.""" + return {k: v[:3] for k, v in self.lens_scan(pr)[0].items()} + + def lens_scan(self, pr): + """(entries, history). entries: {lens key: ((when, index), sha or None, why or None, post)}: each lens's NEWEST + header, from the posts that count (counts). A comment that looks like a lens report but is not one valid read + (lens_problem) is an entry of its own, keyed by the comment, with no sha: a correct new post of that lens does + not replace it, and hiding it does not clear it. It reads nothing until it is deleted, its first line is edited + to `not a lens: ...` with no lens header line left in it, or a newer override covers it (NO_HEADER_WAY_OUT). A + lens comment edited after posting (includesCreatedEdit not false: GitHub records that it changed, not what) or + hidden (minimized) reads nothing until that lens posts a new comment; so does a review whose flags were not read + (FLAGS_UNREAD). A lens-looking post that does not count is logged (log only, never a comment), so a wrong + lens_author_associations shows in the run. review_judge.judge_rounds reads the newest report's findings from the + entry's post. history: [(lens name, sha or None)] of EVERY counted post with a valid lens header, in list order, + older rounds and edited or hidden ones too (review_judge.judge_rounds counts the rounds by it: hiding an old + round must not lower the count).""" + found, history = {}, [] for when, i, post in self.lens_posts(pr): where = post.get('url') or f'by {(post.get("author") or {}).get("login") or "?"} at {when or "?"}' if not self.counts(post): # the author first: a stranger's text is only ever read in part, for the log @@ -462,16 +469,17 @@ def lens_entries(self, pr): header = None if problem else lens_header(post.get('body') or '') if problem: hidden = ', hidden' if self.hidden(post) else '' - found[f'{NO_HEADER} ({where}, {problem}{hidden})'] = ((when, i), None, None) + found[f'{NO_HEADER} ({where}, {problem}{hidden})'] = ((when, i), None, None, post) continue if not header: continue + history.append(header) key = lens_key(header[0]) if key not in found or (when, i) >= found[key][0]: why = FLAGS_UNREAD if post.get(UNREAD) else HIDDEN if self.hidden(post) \ else EDITED if post.get('includesCreatedEdit') is not False else None - found[key] = ((when, i), None, why) if why else ((when, i), header[1], None) - return found + found[key] = ((when, i), None, why, post) if why else ((when, i), header[1], None, post) + return found, history def lens_reads(self, pr): """{lens key: (sha, gap, override)}: what each lens of lens_entries counts as reading, the one rule reviewed_head diff --git a/fork-ci/actions/pr-queue/review_judge.py b/fork-ci/actions/pr-queue/review_judge.py index 6b2600bed1e56..ec49f0b060e5c 100644 --- a/fork-ci/actions/pr-queue/review_judge.py +++ b/fork-ci/actions/pr-queue/review_judge.py @@ -16,9 +16,21 @@ Unknown never judges: a path no rule matches, a protected path with no critical rule (the parser refuses both, so only a hand-built rule set gets here) or a diff line this module cannot read raises JudgeError, which the gate turns into a failure. + +judge_rounds judges the review itself, from the lens comments (README.md, "Review rules", Findings and rounds): + - Findings (`findings: true`): a lens whose newest report that counts as a read has a `blocker:` or `major:` line + fails the head, until that lens reads a fixed head in a new comment without one (a later round clears it) or a + person's override comment newer than the report covers it. Owner decision: issue #99. + - Rounds: each different full sha the lens headers name is one fix-and-review round, and a header's round marker + (`r3`) counts as that round too, whichever is higher. From round `rounds.approach_at` the approach lens is + required; a round past `rounds.max` fails the head until a person's override covers every lens: no more rounds, + the person decides (override or close). Owner decision: issue #100. +Pure as judge is: the lens comments come in (lens.Review.lens_scan), the verdict goes out. """ +import re from dataclasses import dataclass +import lens import review_config import review_glob @@ -152,3 +164,89 @@ def judge(review, changes, config_path): lenses = tuple(dict.fromkeys(n for r in review.rules if r.weight == weight and any(h.rule == r.name for h in hits) for n in r.lenses)) return Judgement(weight, lenses, hits, tuple(protected), len(paths)) + + +# A finding line, as a lens report writes it (README.md, "Findings and rounds"): `blocker:` or `major:` at the start +# of a line, after the Markdown lens.norm_line strips (quote and list markers, emphasis, headings), any case, a plural, +# a number and a short note in brackets allowed (`Majors:`, `major 2:`, `major (state):`). Every line counts, in code +# blocks and quotes too: fail closed, as for the lens header. Only an explicit nothing after the colon (NO_FINDING) is +# no finding; any other text is one. +FINDING_RE = re.compile(r'(blocker|major)s?(?:\s*#?\d{1,4})?(?:\s*\([^()]{0,80}\))?\s*:(.*)', re.I) +NO_FINDING_RE = re.compile(r'(?:none|nothing|no|0|n/?a|-+|\u2014|\u2013)\s*\.?', re.I) +ROUND_RE = re.compile(r'r(\d{1,9})') +SHOW_FINDING = 120 # characters of a finding line named in the reason + + +def findings(body): + """[(kind, line)] of the report's blocker and major lines, in order: kind is 'blocker' or 'major', line the line as + normalised (cut to SHOW_FINDING). Read both ways lens reads a line (clean_inline narrow or not): a line either way + reads as a finding is one.""" + out = [] + for raw in (body or '').splitlines(): + for narrow in (False, True): + m = FINDING_RE.match(lens.norm_line(raw, narrow)) + if m and not NO_FINDING_RE.fullmatch(m.group(2).strip()): + out.append((m.group(1).lower(), lens.norm_line(raw, narrow)[:SHOW_FINDING])) + break + return out + + +def round_marker(name): + """The highest round marker (`r3`) among the words of a lens name after its first, or 0. Emphasis around a word + does not count (`data **r3**`).""" + words = [w.strip('*_`') for w in name.lower().split()[1:]] + return max((int(m.group(1)) for w in words for m in [ROUND_RE.fullmatch(w)] if m), default=0) + + +def review_round(history): + """The fix-and-review round the lens headers say: how many different full 40-hex shas they read, or the highest + round marker, whichever is higher. history: [(lens name, sha or None)] of every lens header that counts + (lens.Review.lens_scan), old, edited and hidden ones too. 0: no lens header yet.""" + shas = {sha for _, sha in history if sha and lens.FULL_SHA.fullmatch(sha)} + return max([len(shas)] + [round_marker(name) for name, _ in history]) + + +@dataclass(frozen=True) +class Rounds: + round: int # review_round of the lens headers + approach: bool # the approach lens is required (round >= rounds.approach_at) + over_cap: bool # round > rounds.max and no override covers every lens + findings: tuple # (lens key, kind, line) of every blocker or major no later round or override cleared + problems: tuple # why the head fails, plain text; empty: findings and rounds pass + + def text(self, review): + """One line for the check summary.""" + cap = review.rounds.max + said = f'round {self.round} of {cap}' + if self.approach: + said += f', approach lens {review.rounds.approach_lens} required from round {review.rounds.approach_at}' + if not review.findings: + said += '; findings are not judged (findings: false)' + return said + ('; ' + '; '.join(self.problems) if self.problems else '; no blocker or major open') + + +def judge_rounds(review, history, newest, overridden): + """Rounds of a pull request's lens comments under the parsed `review` rules. history: [(lens name, sha)] of every + lens header that counts (lens.Review.lens_scan); newest: {lens key: body} of every lens (lens.Review.lens_reads), + the body of its newest report when that report counts as a read, else None (it reads nothing: the head fails for + it already, so it has no findings here); overridden: the lens keys a newer override covers (lens_reads).""" + n, cap = review_round(history), review.rounds.max + every = bool(newest) and set(newest) <= set(overridden) # a person's override covers every lens + over = n > cap and not every + decides = 'a person decides: an override comment for the head, or close the pull request' + problems = [] + if over: + problems.append(f'review round {n} is past the cap of {cap} rounds (issue #100): no more rounds; {decides}') + found = [] + if review.findings: + for key in sorted(newest): + if key in overridden or newest[key] is None: + continue + for kind, line in findings(newest[key]): + found.append((key, kind, line)) + for key, kind, line in found: + way = (f'this is round {n} of {cap}: no more rounds; {decides}' if n >= cap else + 'fix it and have that lens read the fixed head in a new comment with no blocker or major line, or a ' + 'person posts an override comment for the head') + problems.append(f'lens {key} reported a {kind} that no later round cleared (issue #99): {line!r}; {way}') + return Rounds(n, n >= review.rounds.approach_at, over, tuple(found), tuple(problems)) diff --git a/fork-ci/actions/pr-queue/review_judge.test.py b/fork-ci/actions/pr-queue/review_judge.test.py index d154b1a5d4332..0cd2c485c3dac 100644 --- a/fork-ci/actions/pr-queue/review_judge.test.py +++ b/fork-ci/actions/pr-queue/review_judge.test.py @@ -163,5 +163,150 @@ def test_the_parser_refuses_those_rule_sets_first(self): rc.parse_review({**block, 'rules': rules}) +# --- findings and rounds (issues #99, #100): judge_rounds on lens comments --------------------------------------- +H1, H2, H3, H4, H5 = (c * 40 for c in 'abcde') +BASIC = CONFIGS['basic'] # findings true, rounds approach_at 3, max 4, approach lens `approach` + + +def rounds(history, newest, overridden=(), review=BASIC): + return rj.judge_rounds(review, list(history), dict(newest), set(overridden)) + + +class Findings(unittest.TestCase): + def test_blocker_and_major_lines_are_findings_minor_ones_are_not(self): + body = ('lens: code · reviewer: x · read at: ' + H1 + '\n' + 'blocker: the lock is gone\n- **Major:** two writers race\nminor: a typo\n### major 2: a retry loops\n' + 'Majors (state): stopped half-way leaves a row\n> major: quoted from round 1\n```\nmajor: in code\n```') + self.assertEqual([k for k, _ in rj.findings(body)], ['blocker', 'major', 'major', 'major', 'major', 'major']) + self.assertEqual(rj.findings(body)[0], ('blocker', 'blocker: the lock is gone')) + + def test_only_an_explicit_nothing_is_no_finding(self): + for line in ('major: none', 'Major: None.', 'blocker: nothing', '**blockers:** no', 'major: 0', 'major: n/a', + 'major: NA', 'major: -', 'major: —', '- majors: none'): + with self.subTest(line): + self.assertEqual(rj.findings(line), []) + for line in ('major:', 'major: no lock on the write path', 'major: none of the retries stop', 'major: - x', + 'blocker: see below'): + with self.subTest(line): + self.assertEqual(len(rj.findings(line)), 1) + + def test_a_line_that_only_mentions_a_finding_is_none(self): + for line in ('fixed: the major of round 1', 'no major found', 'majority: x', 'Major version: 2', 'blockerx: y', + 'major 12345: too long a number'): + with self.subTest(line): + self.assertEqual(rj.findings(line), []) + + def test_html_and_links_cannot_hide_a_finding(self): # read both ways, as the lens header is + for line in ('major: x', '[major](http://e/x): y', '
major: x', + 'major (see '): # the last only the narrow read finds: the wide tag eats the colon + with self.subTest(line): + self.assertEqual(len(rj.findings(line)), 1, line) + + +class Rounds(unittest.TestCase): + def test_each_different_full_sha_read_is_a_round(self): + self.assertEqual(rj.review_round([]), 0) + self.assertEqual(rj.review_round([('code', H1), ('data-1', H1)]), 1) + self.assertEqual(rj.review_round([('code', H1), ('code', H2), ('data-1', H2), ('code', H3)]), 3) + self.assertEqual(rj.review_round([('code', 'abcdef1'), ('code', None)]), 0) # a short sha or none: no round + + def test_a_round_marker_counts_when_it_is_higher(self): # deleted old rounds: the marker still says it + self.assertEqual(rj.review_round([('code r3', H1)]), 3) + self.assertEqual(rj.review_round([('code **r4**', H1), ('data r2', H2)]), 4) + self.assertEqual(rj.review_round([('code r1', H1), ('code r1', H2), ('code', H3)]), 3) + self.assertEqual(rj.round_marker('r2d2 lens'), 0) + self.assertEqual(rj.round_marker('r9'), 0) # a name's first word is its name, never a marker + + def test_no_lens_comment_yet_is_round_zero_and_passes(self): # empty state + r = rounds([], {}) + self.assertEqual((r.round, r.approach, r.over_cap, r.findings, r.problems), (0, False, False, (), ())) + + def test_a_major_blocks_until_a_later_round_of_that_lens_clears_it(self): + r = rounds([('code', H1)], {'code': 'major: two writers race'}) + self.assertEqual(r.findings, (('code', 'major', 'major: two writers race'),)) + self.assertIn('lens code reported a major that no later round cleared', r.problems[0]) + self.assertIn('read the fixed head in a new comment', r.problems[0]) + r = rounds([('code', H1), ('code r2', H2)], {'code': 'major: none'}) # round 2 cleared it + self.assertEqual((r.round, r.problems), (2, ())) + + def test_an_override_newer_than_the_report_clears_its_findings(self): # #99: only a person's override + r = rounds([('code', H1), ('data-1', H1)], {'code': 'blocker: x', 'data-1': 'major: y'}, overridden={'code'}) + self.assertEqual([f[0] for f in r.findings], ['data-1']) + r = rounds([('code', H1)], {'code': 'blocker: x'}, overridden={'code'}) + self.assertEqual(r.problems, ()) + + def test_a_lens_that_reads_nothing_is_not_judged_here(self): # it fails the head already (edited, hidden) + r = rounds([('code', H1)], {'code': None}) + self.assertEqual(r.problems, ()) + + def test_findings_false_judges_no_finding(self): + r = rounds([('code', H1)], {'code': 'blocker: x'}, review=dataclasses.replace(BASIC, findings=False)) + self.assertEqual(r.problems, ()) + self.assertIn('findings are not judged', r.text(dataclasses.replace(BASIC, findings=False))) + + def test_the_approach_lens_is_required_from_round_approach_at(self): + self.assertFalse(rounds([('code', H1), ('code', H2)], {'code': ''}).approach) + r = rounds([('code', H1), ('code', H2), ('code', H3)], {'code': ''}) + self.assertTrue(r.approach) + self.assertIn('approach lens approach required from round 3', r.text(BASIC)) + + def test_a_round_past_the_cap_stops_and_a_person_decides(self): # #100 + four = [('code', h) for h in (H1, H2, H3, H4)] + self.assertEqual(rounds(four, {'code': ''}).problems, ()) # round 4 of 4 may pass + r = rounds(four + [('code', H5)], {'code': ''}) + self.assertTrue(r.over_cap) + self.assertEqual(r.problems, ('review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a ' + 'person decides: an override comment for the head, or close the pull request',)) + self.assertEqual(rounds([('code r5', H1)], {'code': ''}).round, 5) + + def test_an_override_covering_every_lens_passes_the_cap_a_newer_round_does_not(self): + five = [('code', h) for h in (H1, H2, H3, H4, H5)] + [('approach', H5)] + self.assertEqual(rounds(five, {'code': '', 'approach': ''}, overridden={'code', 'approach'}).problems, ()) + r = rounds(five, {'code': '', 'approach': ''}, overridden={'code'}) # approach posted after the override + self.assertTrue(r.over_cap) + + def test_a_finding_at_the_last_round_says_no_more_rounds(self): + r = rounds([('code', h) for h in (H1, H2, H3, H4)], {'code': 'major: still racy'}) + self.assertIn('this is round 4 of 4: no more rounds; a person decides', r.problems[0]) + + def test_the_same_comments_give_the_same_verdict(self): # runs again: pure + args = ([('code', H1), ('code r2', H2)], {'code': 'major: x'}, {'data-1'}) + self.assertEqual(rounds(*args), rounds(*args)) + + +# --- the fork's own config (issue #730): in enforce, workflow, queue and lens files critical (#82) ---------------- +ROOT = os.path.join(HERE, '..', '..', '..') +FORK_CONFIG = '.github/fork-pr-queue.json' + + +class ForkConfig(unittest.TestCase): + def setUp(self): + with open(os.path.join(ROOT, FORK_CONFIG), encoding='utf-8') as f: + self.cfg = config.parse(f.read(), FORK_CONFIG) + self.review = self.cfg.review + + def test_it_parses_in_enforce_with_the_owners_numbers(self): + r = self.review + self.assertEqual((r.mode, r.findings, r.rounds.approach_at, r.rounds.max, r.warnings), + ('enforce', True, 3, 4, ())) + self.assertGreaterEqual(r.min_lenses.critical, r.min_lenses.normal + 1) # #82: one extra review + for x in r.lenses: + with self.subTest(x.name): + self.assertTrue(os.path.isfile(os.path.join(ROOT, x.prompt)), x.prompt) + + def test_workflow_queue_gate_config_and_lens_files_are_critical(self): # #82 + critical = next(x for x in self.review.rules if x.weight == 'critical') + for path in ('.github/workflows/fork-head-gate.yml', 'fork-ci/actions/pr-queue/review_judge.py', + 'fork-ci/actions/head-check/head_check.py', FORK_CONFIG, '.github/fork-lens-state.md', + '.github/actions/setup-ci-deps/action.yml', '.github/fork-ci-release-rulesets.json'): + with self.subTest(path): + j = rj.judge(self.review, [('M', (path,))], FORK_CONFIG) + self.assertEqual((j.weight, j.lenses), ('critical', critical.lenses)) + self.assertGreaterEqual(len(j.lenses), 2) + j = rj.judge(self.review, [('M', ('apps/server/src/x.ts',))], FORK_CONFIG) + self.assertEqual(j.weight, 'normal') + self.assertTrue(j.lenses) + + if __name__ == '__main__': unittest.main() From d7380fb62a05ac84f8321ee2c8d5ff72b2dcf6e6 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:10:07 +0000 Subject: [PATCH 2/3] fork ci: read lens findings both ways, count fix rounds, person-only overrides (#740 fix) Findings (#99): headings, bold, tables, dash and emoji labels count, after NFKC, zero-width removal and o/0/M look-alikes; clean lines (none found, none!, none (only minors)) and fenced code do not. A report on the head with no findings section fails that lens instead of passing as clean. NOT SAFE on the title or a verdict line is a blocker. Rounds (#100): a round is a different change of the pull request's own diff (from its merge base), so a queue update that only merges the base in adds none. Edited, hidden and short-sha reads still count; a round marker in a lens name is a floor. Past the cap the message says a person's override alone is enough, also for a missing approach lens. Overrides (#78): review.override_authors lists the people whose override counts; empty or left out means none is accepted. The fork config stays in shadow and leaves the list out, and the lens prompts use the exact header and findings section the gate reads. Co-Authored-By: Claude Opus 5.5 --- .github/fork-lens-approach.md | 19 +- .github/fork-lens-code.md | 19 +- .github/fork-lens-merge-path.md | 19 +- .github/fork-lens-state.md | 19 +- .github/fork-pr-queue.json | 2 +- fork-ci/actions/head-check/README.md | 20 +- fork-ci/actions/head-check/head_check.py | 93 +++++- fork-ci/actions/head-check/head_check.test.py | 87 ++++- fork-ci/actions/pr-queue/README.md | 107 +++--- fork-ci/actions/pr-queue/review_config.py | 33 +- fork-ci/actions/pr-queue/review_judge.py | 304 ++++++++++++++---- fork-ci/actions/pr-queue/review_judge.test.py | 256 +++++++++++---- 12 files changed, 779 insertions(+), 199 deletions(-) diff --git a/.github/fork-lens-approach.md b/.github/fork-lens-approach.md index ebbb664dca9bb..cc7ed4d990ef6 100644 --- a/.github/fork-lens-approach.md +++ b/.github/fork-lens-approach.md @@ -7,7 +7,18 @@ more small findings. Ask whether the whole approach is right: - Is there a simpler way, or an existing mechanism it should build on? - Will more rounds converge, or should it be split, restarted or closed? -Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections -"Lens comments" and "Findings and rounds"): the first line is the lens header, -then one line per finding that starts `blocker:`, `major:` or `minor:`. -Write `major: none` when there is none. +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: approach · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-lens-code.md b/.github/fork-lens-code.md index 4e7fa12580eb9..6f66fc3d8ca81 100644 --- a/.github/fork-lens-code.md +++ b/.github/fork-lens-code.md @@ -8,7 +8,18 @@ simple as it can be? Read the diff and the files it touches; run its tests. old or empty state? - Tests: do they cover the change, and do they fail on the old code? -Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections -"Lens comments" and "Findings and rounds"): the first line is the lens header, -then one line per finding that starts `blocker:`, `major:` or `minor:`. -Write `major: none` when there is none. +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: code · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-lens-merge-path.md b/.github/fork-lens-merge-path.md index 2a02ca3c5a394..2f0992994eaf7 100644 --- a/.github/fork-lens-merge-path.md +++ b/.github/fork-lens-merge-path.md @@ -12,7 +12,18 @@ the files it touches; run its tests. - This repository is public: nothing personal (names, addresses, paths of a person's machine, tokens) in files, commits or comments. -Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections -"Lens comments" and "Findings and rounds"): the first line is the lens header, -then one line per finding that starts `blocker:`, `major:` or `minor:`. -Write `major: none` when there is none. +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: merge-path · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-lens-state.md b/.github/fork-lens-state.md index 157fad02f6a05..c7867fa163675 100644 --- a/.github/fork-lens-state.md +++ b/.github/fork-lens-state.md @@ -9,7 +9,18 @@ files it touches; run its tests. - Each fix line should turn a test red when it is removed. Spot-check some. - New tests should fail on the old code. Check one. -Write the report as one comment (fork-ci/actions/pr-queue/README.md, sections -"Lens comments" and "Findings and rounds"): the first line is the lens header, -then one line per finding that starts `blocker:`, `major:` or `minor:`. -Write `major: none` when there is none. +Write the report as one comment in the pull request's conversation, never as +inline review comments: findings there are not read. Its first line is exactly + + lens: state · reviewer: · read at: + +Then the findings section, always, also when it is empty: + + blocker: none + major: none + +or, for each finding, one line that starts `blocker:` or `major:` (`minor:` +for the rest). Quote code or example findings only inside a fenced code block. +A report without its `blocker:` and `major:` lines cannot be judged and fails +the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings +and rounds"). diff --git a/.github/fork-pr-queue.json b/.github/fork-pr-queue.json index 65e4da643face..a77cb870e66f9 100644 --- a/.github/fork-pr-queue.json +++ b/.github/fork-pr-queue.json @@ -15,7 +15,7 @@ "head_check": "head_check", "review": { "version": 1, - "mode": "enforce", + "mode": "shadow", "rules": [ { "name": "gate", diff --git a/fork-ci/actions/head-check/README.md b/fork-ci/actions/head-check/README.md index 6b7d338f684b5..3571169aa8b76 100644 --- a/fork-ci/actions/head-check/README.md +++ b/fork-ci/actions/head-check/README.md @@ -52,13 +52,19 @@ All of them are the queue's, imported from `../pr-queue`, never copied: Without the block, nothing of this runs. - **Findings and rounds** (`review_judge.judge_rounds`, the pr-queue README, "Findings and rounds"): with a `review` block in `enforce`, a lens whose - newest report has a `blocker:` or `major:` line fails the head until a later - round of that lens clears it or a person's override covers it - (`findings: true`); from round `rounds.approach_at` the approach lens is - required; a round past `rounds.max` fails the head until a person's override - covers every lens (no more rounds: the person overrides or closes). In - `shadow` the summary only adds a row "they would have said: ...". Comments - that cannot be judged fail the head, in either mode. + newest report has a blocker or major finding (a `blocker:` or `major:` line, + a `### Blocking` or `### Major` heading with items, a table row ...) fails + the head until a later round of that lens clears it or a person's override + newer than the report covers it (`findings: true`); a report with no findings + section at all fails too. A person is a login in `override_authors`; none + listed (the default) means no override clears a finding or passes the cap + (issue #78). A round is a read of a different own change (`own_change`: a + head that only merges the base in is no new round); from round + `rounds.approach_at` the approach lens is required; a round past + `rounds.max` fails the head until a person's override for it, newer than + every lens comment, lets it through on its own. In `shadow` the summary only + adds a row "they would have said: ...". Comments that cannot be judged fail + the head, in either mode. It passes when: diff --git a/fork-ci/actions/head-check/head_check.py b/fork-ci/actions/head-check/head_check.py index 3a0fa7d6eb731..8457260fcabe1 100644 --- a/fork-ci/actions/head-check/head_check.py +++ b/fork-ci/actions/head-check/head_check.py @@ -20,10 +20,12 @@ are `git diff --name-status -M` from the merge base to the head (changed_paths), judged by review_judge.judge. In `enforce` its lenses are required too; in `shadow` the summary only says what they would have been. A change it cannot judge (no merge base, a failed diff) is refused, never passed. The same block judges the review itself -(review_judge.judge_rounds, review_rounds): in `enforce` a lens's newest report with a `blocker:` or `major:` line that -no later round or override cleared fails the head (`findings: true`), the approach lens is required from round -`rounds.approach_at`, and a round past `rounds.max` fails the head until a person's override covers every lens; in -`shadow` the summary only says so. Without the block nothing of this runs. +(review_judge.judge_rounds, review_rounds): in `enforce` a lens's newest report with a blocker or major finding that +no later round or person cleared fails the head (`findings: true`), and so does a report with no findings section; +the approach lens is required from round `rounds.approach_at`; a round past `rounds.max` fails the head until a +person's override comment for it (a login in `override_authors`; none listed: no override is a person's, issue #78) +lets it through. A round is a read of a different own change (own_change), so a queue update adds none. In `shadow` +the summary only says so. Without the block nothing of this runs. Pass: every lens read the head (or a newer override vouches for it), and every required lens is there; a pull request with no lens comments and no required lens (nothing to compare); a draft, with --pass-drafts only. Fail, @@ -42,7 +44,7 @@ head come from differs (Checkout). `--required-lenses` takes one lens name per line (a name may hold a comma: `data-integrity (DI-1, DI-2)`). """ -import argparse, json, os, re, subprocess, sys, tempfile, time, unicodedata +import argparse, hashlib, json, os, re, subprocess, sys, tempfile, time, unicodedata sys.dont_write_bytecode = True # no __pycache__ next to the scripts sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'pr-queue')) @@ -350,7 +352,7 @@ def decide(repo, n, config_path, checkout, required, pass_drafts, gh, sleep, out log=lambda text: out(escape(text))) rounds = None if paths is not None and not (pr['isDraft'] and pass_drafts): - rounds, why = review_rounds(cfg.review, review, pr) + rounds, why = review_rounds(cfg.review, review, pr, checkout, cfg.override_prefix) if rounds is None: # fail closed, in shadow as in enforce, as for the path rules return 1, False, f'the review rounds and findings cannot be judged, so the head is not passed: {why}', [row] if enforced and rounds.approach: @@ -369,17 +371,80 @@ def decide(repo, n, config_path, checkout, required, pass_drafts, gh, sleep, out return (0 if ok else 1), ok, reason, rows -def review_rounds(rules, review, pr): +def own_change(checkout, base): + """change_of for review_judge.review_round: what the pull request itself changes at a read commit, so two reads of + the same change are one round. The key is a hash of `git diff -U0` from the commit's merge base with `base` to it, + hunk line numbers dropped: a head that only merges the base in (a queue or branch update) has the same own change + as the head it updates, and is no new round. A short sha is resolved when it names one commit. A commit that is not + in the repository and cannot be fetched, more than one merge base or a failed diff: a round of its own (unknown + counts more rounds, never fewer).""" + cache = {} + + def of(sha): + if sha not in cache: + cache[sha] = diff_key(checkout, base, sha) or ('unknown', sha) + return cache[sha] + return of + + +def diff_key(checkout, base, sha): + if lens.FULL_SHA.fullmatch(sha): + if not (checkout.has(sha) or checkout.fetch(sha)): + return None + full = sha + else: + r = checkout.git('rev-parse', '-q', '--verify', '--end-of-options', sha + '^{commit}') + full = r.stdout.strip() if r.returncode == 0 else '' + if not SHA.fullmatch(full): + return None + r = checkout.git('merge-base', '--all', base, full) + bases = r.stdout.split() if r.returncode == 0 else [] + if len(bases) != 1 or not SHA.fullmatch(bases[0]): + return None + d = checkout.git('diff', '--no-ext-diff', '--no-textconv', '--no-relative', '--ignore-submodules=none', + '--no-color', '--no-renames', '--full-index', '-U0', bases[0], full, '--', + errors='surrogateescape') + if d.returncode != 0: + return None + text = re.sub(r'^@@ [^@]* @@', '@@', d.stdout, flags=re.M) + return ('change', hashlib.sha256(text.encode('utf-8', 'surrogateescape')).hexdigest()) + + +def review_rounds(rules, review, pr, checkout, prefix): """(review_judge.Rounds, None) of the pull request's lens comments under the base's `review` rules, or (None, why). - Findings come from each lens's newest report that counts as a read; the round from every lens header that counts. - Any error is a why, never a pass.""" + The round counts every lens header the check reads now (lens_scan's history: old, edited and hidden ones too; + a deleted comment is gone from GitHub, so only the round marker of a later header keeps it counted), each read + change once (own_change). Each lens's newest entry is judged as review_judge says (REPORT ...): a person's override + (an override comment by a login in review.override_authors that vouches for this head: its sha or that change + merged with the base) newer than the entry clears it; any other override clears no finding. Any error is a why, + never a pass.""" try: + head = str(pr.get('headRefOid') or '').lower() + parents = checkout.merge_parents() + if not parents: + return None, "HEAD is not GitHub's two-parent merge commit, so the rounds are unknown" entries, history = review.lens_scan(pr) - reads = review.lens_reads(pr) - newest = {k: (entries[k][3].get('body') or '') if sha and not override else None - for k, (sha, _, override) in reads.items()} - overridden = {k for k, (_, _, override) in reads.items() if override} - return review_judge.judge_rounds(rules, history, newest, overridden), None + people = {a.lower() for a in rules.override_authors} + overrides = [((when, i), sha, post) for when, i, post in review.lens_posts(pr) + for sha in [review.override_sha(post)] if sha] + newest = max((at for at, _, _ in overrides), default=None) + person = max((at for at, sha, post in overrides + if ((post.get('author') or {}).get('login') or '').lower() in people + and (sha == head or same_change(checkout, head, sha)[0])), default=None) + reports = {} + for key, (at, sha, _, post) in entries.items(): + if person is not None and at < person: + how = review_judge.CLEARED + elif key.startswith(lens.NO_HEADER): + how = review_judge.LOOSE if newest is not None and at < newest else review_judge.NOT_JUDGED + elif lens.FULL_SHA.fullmatch(sha or ''): + how = review_judge.REPORT + else: + how = review_judge.UNREAD if newest is not None and at < newest else review_judge.NOT_JUDGED + reports[key] = (how, post.get('body') or '') + decided = person is not None and all(at < person for at, _, _, _ in entries.values()) + return review_judge.judge_rounds(rules, history, reports, decided, own_change(checkout, parents[0]), + prefix), None except Exception as e: # noqa: BLE001 - unknown never passes return None, f'{type(e).__name__}: {review_judge.shown(str(e))[:300]}' diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index f2d31225d6d78..5132fa0a6b2bd 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -1285,11 +1285,15 @@ class FindingsAndRounds(unittest.TestCase): setUp, base, head, check = PathRules.setUp, PathRules.base, PathRules.head, PathRules.check def said(self, name, sha, *lines, **kw): + """A lens report; with no lines a clean one (`blocker: none`, `major: none`).""" + lines = lines or ('blocker: none', 'major: none') return self.p.post(f'{LENS}: {name} · reviewer: x · read at: {sha}\n' + '\n'.join(lines), **kw) - def pair(self, mode='enforce'): - b = self.base(mode) - return b, self.head(b, mode, **{'src/app.py': 'a\n'}) + def pair(self, mode='enforce', **change): + """A base with the review block (changed by `change`) and a head on it that changes one normal path.""" + cfg = review_cfg(mode, **change) + b = self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE}), [self.m0]) + return b, self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.py': 'a\n'}), [b]) def test_a_major_in_the_newest_report_fails_the_head(self): base, head = self.pair() @@ -1348,26 +1352,97 @@ def test_the_approach_lens_is_required_from_round_three(self): self.assertEqual(code, 0, out) self.assertIn('round 3 of 4, approach lens approach required from round 3', out) - def test_past_the_cap_it_stops_and_only_an_override_passes(self): + def test_past_the_cap_it_stops_and_only_a_persons_override_passes(self): # #100 base, head = self.pair() five = [self.said('code', sha) for sha in OLD] + [self.said('code', head), self.said('approach', head)] code, out = self.check(base, head, comments=five) self.assertEqual(code, 1, out) self.assertIn('review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a person decides', out) code, out = self.check(base, head, comments=five + [self.p.override(head)]) - self.assertEqual(code, 0, out) + self.assertEqual(code, 1, out) # no override_authors (the default, #78): the owner's login proves no person + self.assertIn('no one is listed in review.override_authors yet, so no override is accepted', out) + base, head = self.pair(override_authors=['owner']) + five = [self.said('code', sha) for sha in OLD] + [self.said('code', head)] # no approach lens at all + code, out = self.check(base, head, comments=five) + self.assertEqual(code, 1, out) + self.assertIn('an override comment `tool ready --head : ` for this head by owner', out) + code, out = self.check(base, head, comments=five + [self.p.override(head)]) + self.assertEqual(code, 0, out) # the override alone is enough: the approach lens is waived too stranger = self.p.override(head, assoc='CONTRIBUTOR', login='someone') # not the owner's: counts for nothing self.assertEqual(self.check(base, head, comments=five + [stranger])[0], 1) + other = self.p.override(head, login='agent-login') # the right association, a login not listed + self.assertEqual(self.check(base, head, comments=five + [other])[0], 1) + override = self.p.override(head) + later = self.said('code', head) # a lens comment newer than the override: the person decided before it + self.assertEqual(self.check(base, head, comments=five + [override, later])[0], 1) - def test_an_override_clears_a_wrong_finding(self): # #99: a person's override comment + def test_only_a_listed_persons_override_clears_a_finding(self): # #99, #78 base, head = self.pair() report = self.said('code', head, 'blocker: wrong, the lock is there') self.assertEqual(self.check(base, head, comments=[report])[0], 1) + code, out = self.check(base, head, comments=[report, self.p.override(head)]) + self.assertEqual(code, 1, out) # an override from a login agents still post as clears nothing + self.assertIn("lens code reported a blocker that no later round cleared (issue #99): 'blocker: wrong", out) + base, head = self.pair(override_authors=['Owner']) # logins compare case-insensitively + report = self.said('code', head, 'blocker: wrong, the lock is there') self.assertEqual(self.check(base, head, comments=[report, self.p.override(head)])[0], 0) override = self.p.override(head) later = self.said('code r2', head, 'blocker: wrong again') # a report newer than the override counts again self.assertEqual(self.check(base, head, comments=[report, override, later])[0], 1) + def test_an_unreadable_report_under_an_agents_override_fails(self): # hidden after the override covered it + base, head = self.pair() + hidden = self.said('code', head, 'blocker: x', hidden=True) + code, out = self.check(base, head, comments=[hidden, self.p.override(head)]) + self.assertEqual(code, 1, out) + self.assertIn('lens code: its newest report reads nothing and only an override that is no person', out) + base, head = self.pair(override_authors=['owner']) + hidden = self.said('code', head, 'blocker: x', hidden=True) + self.assertEqual(self.check(base, head, comments=[hidden, self.p.override(head)])[0], 0) + + def test_a_report_with_no_findings_section_fails_the_head(self): # #99: never passes as clean + base, head = self.pair() + code, out = self.check(base, head, comments=[self.said('code', head, 'Looks good to me.')]) + self.assertEqual(code, 1, out) + self.assertIn('lens code: its newest report has no findings section, so it cannot be judged (issue #99)', out) + self.assertEqual(self.check(base, head, comments=[self.said('code', head, '### Major', 'None.')])[0], 0) + + def test_a_finding_in_a_heading_or_table_or_lookalike_fails_the_head(self): # #99, both ways + base, head = self.pair() + for lines in (('### Blocking', '1. **The lock is gone.**'), ('| major | two writers race |',), + ('maj\u043er: x',), ('Blocker: none, major: x',)): + with self.subTest(lines): + self.assertEqual(self.check(base, head, comments=[self.said('code', head, *lines)])[0], 1) + clean = ('blocker: none', '```', 'major: quoted from a test vector', '```', 'Major: none found') + self.assertEqual(self.check(base, head, comments=[self.said('code', head, *clean)])[0], 0) + + def test_a_queue_update_is_no_new_round(self): # #100: a round is a read after a fix, not a new sha + base, h1 = self.pair() + cfg = review_cfg('enforce') + tree = dict(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.py': 'a\nb\n'})) + h2 = self.w.commit(tree, [h1]) # the fix: round 2 + base2 = self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'docs/d.md': 'd\n'}), [base]) + h3 = self.w.merge(h2, base2) # the queue merges the base in: the same change, no round + reads = [self.said('code', h1), self.said('code r2', h2), self.said('code', h3)] + code, out = self.check(base2, h3, comments=reads) + self.assertEqual(code, 0, out) # round 3 would need the approach lens + self.assertIn('round 2 of 4; no blocker or major open', out) + again = self.w.commit({**tree, 'docs/d.md': 'd\n'}, [base2]) # the fix rebased onto the new base: the same + code, out = self.check(base2, again, comments=reads[:2] + [self.said('code', again)]) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4', out) + fixed = self.w.merge(h2, base2, files={**tree, 'docs/d.md': 'd\n', 'src/app.py': 'a\nc\n'}) # a fix in it + code, out = self.check(base2, fixed, comments=reads[:2] + [self.said('code', fixed)]) + self.assertEqual(code, 1, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + + def test_a_short_sha_read_still_counts_a_round(self): # old state: a lens that wrote 12 characters + base, head = self.pair() + short = self.said('code', OLD[0][:12]) + code, out = self.check(base, head, comments=[short, self.said('code', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4', out) + def test_a_strangers_finding_neither_blocks_nor_counts_a_round(self): # anyone can comment on a public repo base, head = self.pair() noise = [self.said('code', sha, 'blocker: x', assoc='NONE', login='someone') for sha in OLD] diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index b586faecd92e4..9a0fd852cf1af 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -68,20 +68,21 @@ yet. `review.recommended.json` (next to this file) is a complete example with made-up paths; a test parses it. Real paths, lens names and prompt files belong in the project's own config (below, "Turning it on"). -Every key is required and an unknown key is an error, as in the rest of the -config. A wrong block makes the whole config invalid, naming every problem at +Every key is required except `override_authors`, and an unknown key is an +error, as in the rest of the config. A wrong block makes the whole config invalid, naming every problem at once. -| Key | Meaning | -| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `version` | `1`. Any other version is refused by name (a newer one: update the pinned release first). | -| `mode` | `shadow`: the gate keeps judging by its `required-lenses` input and only reports what the path rules would have said. `enforce`: the path rules decide. | -| `rules` | An ordered list of `{"name", "weight", "paths", "lenses"}`. A changed file belongs to the first rule with a matching glob. `weight` is `critical`, `normal` or `light`; `lenses` are names from `lenses`. | -| `lenses` | `{"": {"prompt": ""}}`: every lens a rule or `rounds` names, with its prompt file (repo-relative, no `..`, no leading `/`, no glob characters). | -| `min_lenses` | `{"critical", "normal", "light"}`: the fewest different lenses a rule of that weight must list, 0-10, never falling with weight. | -| `rounds` | `{"approach_at", "max", "approach_lens"}`: from round `approach_at` the approach lens is required too; after round `max` a person must write an override comment or close the pull request. `max` is not below `approach_at`. | -| `findings` | `true` or `false`: lens comments carry `blocker:`, `major:` and `minor:` lines, and a blocker or major at the head fails the gate. | -| `state_line` | `true` or `false`: a required lens comment needs a `state:` line to count. Not enforced yet. | +| Key | Meaning | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `version` | `1`. Any other version is refused by name (a newer one: update the pinned release first). | +| `mode` | `shadow`: the gate keeps judging by its `required-lenses` input and only reports what the path rules would have said. `enforce`: the path rules decide. | +| `rules` | An ordered list of `{"name", "weight", "paths", "lenses"}`. A changed file belongs to the first rule with a matching glob. `weight` is `critical`, `normal` or `light`; `lenses` are names from `lenses`. | +| `lenses` | `{"": {"prompt": ""}}`: every lens a rule or `rounds` names, with its prompt file (repo-relative, no `..`, no leading `/`, no glob characters). | +| `min_lenses` | `{"critical", "normal", "light"}`: the fewest different lenses a rule of that weight must list, 0-10, never falling with weight. | +| `rounds` | `{"approach_at", "max", "approach_lens"}`: from round `approach_at` the approach lens is required too; after round `max` a person must write an override comment or close the pull request. `max` is not below `approach_at`. | +| `findings` | `true` or `false`: lens comments carry a findings section (`blocker:` and `major:` lines, or headings), and a blocker or major at the head fails the gate. | +| `override_authors` | Optional, default `[]`: the GitHub logins whose override comment is a person's decision (clears a finding, passes the round cap). `[]`: no override is a person's, so none does that (issue #78). | +| `state_line` | `true` or `false`: a required lens comment needs a `state:` line to count. Not enforced yet. | The config is refused when: @@ -193,29 +194,58 @@ be up to date, which the queue needs anyway. comments it reads anyway (section "Lens comments"; `review_judge.judge_rounds`). Owner decisions: issue #99 (findings), issue #100 (rounds). -- **Findings** (`findings: true`): a lens report writes each finding on a line - of its own that starts `blocker:`, `major:` or `minor:` (any case; Markdown - list, quote and emphasis marks do not count; `Majors:`, `major 2:` and - `major (state):` are the same). A lens whose newest report that counts as a - read has a `blocker:` or `major:` line fails the head. Only an explicit - nothing after the colon is no finding (`major: none`, `nothing`, `no`, `0`, - `n/a`, `-`); any other text is one, also an empty one. Every line counts, in - code blocks and quotes too, so a report names an earlier finding it saw fixed - without that form (`fixed: the major of round 1 ...`). The ways out: fix it, - and the lens posts a new report (a later round: its newest comment counts) - without a blocker or major line; or a person posts an override comment for - the head (config `override`), which covers every lens whose report is older. - `minor:` lines never block. -- **Rounds:** each different full sha that the counted lens headers name is a - fix-and-review round, and a round marker in a lens name (`code r3`) counts as - that round, whichever is higher. Hidden and edited lens comments count too - (hiding old rounds does not lower the count); a deleted one does not, so keep - the round marker in the names. From round `rounds.approach_at` the lens - `rounds.approach_lens` is required as well. Round `rounds.max` is the last: - a round past it fails the head, whatever the lenses said, with "no more - rounds; a person decides", until a person's override comment newer than every - lens comment covers the head; else the pull request is closed. At round - `rounds.max` a blocker or major says the same. +- **Findings** (`findings: true`): a lens report has a findings section in its + body: a `blocker:` and a `major:` line (`blocker: none`, `major: none` when it + found nothing), or headings `### Blocking` / `### Major` with the findings, + or `None.`, under them. What counts as a finding (`review_judge.read_findings`, + table in `review_judge.test.py`): a label `blocker`, `blocking` or `major` + (plural, `Major findings`, `major 2`, `major (state)` too; bold, list, emoji + and heading marks do not matter) followed by text on the same line + (`major: x`, `**Blocking:** x`, `major — x`, `[major] x`, a table row + `| major | x |` or a table column headed `major`), or, for a bare label, the + first line under it (a heading or nothing under it is a finding too). Every + label of a line counts (`blocker: none, major: x`). Only an explicit nothing + is no finding (`none`, `None found`, `none (only minors)`, `nothing`, `no`, + `0`, `n/a`, `-`). Before reading, the text is folded: NFKC, zero-width and + other format characters removed, Cyrillic and Greek look-alike letters read + as Latin, `0` for `o` in a label. Lines inside a fenced code block are never + labels, so quote code and example findings in one; quotes and indented + blocks are read (fail closed). A title line or `verdict:` line that says + NOT SAFE is a blocker. `minor:` and `nit:` never block. + A lens whose newest report has a blocker or major fails the head. A report + with no findings section at all (not even `major: none`) cannot be judged and + fails that lens. Findings go in the report body: findings in inline review + comments (on a line of the diff) are not read, so a lens posted as a PR review + whose body has no findings section fails. The ways out: fix it, and the lens + posts a new report on the fixed head without a blocker or major; or a person + posts an override comment for the head (config `override`) newer than the + report. A person is a login listed in `override_authors`; an override by any + other author of the override association covers a lens's read as before, but + clears no finding and passes no cap. While agents still post as the owner's + login (issue #78) an override from it proves no person, so keep the list + empty until they no longer do: then no override clears a finding, and a + finding is fixed or the pull request closed. +- **Rounds:** a round is a lens read of a fixed change, not of a new sha. The + gate counts the different own changes the counted lens headers read: for each + read sha, `git diff -U0` from its merge base with the base to it, line numbers + dropped (`head_check.own_change`). A head that only merges the base in (a + queue update, `gh pr update-branch`) or the same change rebased onto a newer + base is the same change, so the same round; a merge that also changes the pull + request's own diff is a new one. A short sha counts as well, and a sha the + gate cannot fetch, or one with several merge bases, counts as a round of its + own (unknown counts more, never fewer). A round marker in a lens name + (`code r3`, also `**r3**`) counts as that round when it is higher. The count + comes from the lens comments the gate reads now: hidden and edited ones count + too, so hiding or editing old rounds does not lower it. A deleted comment is + gone from GitHub, and only lens authors can delete one, so the round marker of + a later header is what keeps it counted; counting from commits instead would + count every push, the opposite of a round. From round `rounds.approach_at` + the lens `rounds.approach_lens` is required as well. Round `rounds.max` is the + last: a round past it fails the head with "no more rounds; a person decides": + the pull request is closed, or a person's override comment for the head, + newer than every lens comment, lets it through on its own (it also waives the + approach lens). With `override_authors` empty no override does: the reason + says so. At round `rounds.max` a blocker or major says the same. - In `shadow` the summary adds a row "findings and rounds (shadow): they would have said: ..." and nothing fails. Lens comments that cannot be judged fail the head in both modes. Without the `review` key nothing of this runs. @@ -232,7 +262,9 @@ the fork: add the `review` block to the queue config its head gate reads `shadow` first. Read a few gate summaries, fix every config warning, then set `mode` to `enforce` and dispatch the backfill (head-check README). The fork's own config is [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), -in `enforce`. +in `shadow`, without `override_authors` (no person yet) until issue #78 is done; its lens +prompts (`.github/fork-lens-*.md`) tell each lens the exact header and findings +lines, so a switch to `enforce` does not fail every pull request. **The empty input.** The gate refuses an empty `required-lenses` input (a head with no lens comment would pass), except when the config at the base tip has a @@ -249,7 +281,8 @@ posts a failure that says why, in `shadow` as in `enforce`. Never a pass, never **Adding the block.** A release that does not know the `review` key refuses the whole config (an unknown key), so: release first, then pin that release in every caller of the queue and the gate, then add the key. Roll back in the reverse order: -remove the key, then the pin. +remove the key, then the pin. The same holds for `override_authors`: a +release from before it refuses a config that has it. ## Lens comments diff --git a/fork-ci/actions/pr-queue/review_config.py b/fork-ci/actions/pr-queue/review_config.py index 24612706f48ad..801f3bec5eccf 100644 --- a/fork-ci/actions/pr-queue/review_config.py +++ b/fork-ci/actions/pr-queue/review_config.py @@ -28,6 +28,12 @@ WEIGHTS = ('critical', 'normal', 'light') # heaviest first CATCH_ALL = '**' KEYS = ('version', 'mode', 'rules', 'lenses', 'min_lenses', 'rounds', 'findings', 'state_line') +# Optional keys, with what leaving them out means. override_authors: the GitHub logins whose override comment counts as a +# person's decision (a finding cleared, the round cap passed). Left out or empty: no override is a person's (issue #78: +# while agents post as the owner's login, an override from it proves no person). +OPTIONAL = ('override_authors',) +LOGIN_RE = re.compile(r'[A-Za-z0-9](?:[A-Za-z0-9]|-(?=[A-Za-z0-9])){0,38}') +MAX_OVERRIDE_AUTHORS = 20 RULE_KEYS = ('name', 'weight', 'paths', 'lenses') ROUND_KEYS = ('approach_at', 'max', 'approach_lens') MAX_LENSES = 10 @@ -83,6 +89,7 @@ class ReviewRules: warnings: tuple = () # The rule-order problems themselves, in either mode (enforce_order_problems): empty in a parsed enforce block. order: tuple = () + override_authors: tuple = () # GitHub logins as written, each once (case-insensitive); empty: no person's override def lens(self, name): return next((x for x in self.lenses if x.name == name), None) @@ -447,7 +454,8 @@ def parse_review(obj, where='review'): newer = _int(obj['version'], VERSION + 1, 2 ** 31) raise ConfigError(f'{where}: version {_show(obj["version"])} is not {VERSION}, the only version this release ' f'reads' + (': update the pinned release first' if newer else '')) - problems.extend(f'unknown key {k!r} (known: {", ".join(KEYS)})' for k in obj if k not in KEYS) + problems.extend(f'unknown key {k!r} (known: {", ".join(KEYS + OPTIONAL)})' for k in obj + if k not in KEYS + OPTIONAL) problems.extend(f'{k} is missing' for k in KEYS if k not in obj) if 'mode' in obj and obj['mode'] not in MODES: problems.append(f'mode is {_show(obj["mode"])}: must be one of {", ".join(MODES)}') @@ -461,6 +469,7 @@ def parse_review(obj, where='review'): mins = _min_lenses(obj['min_lenses'], problems) if 'min_lenses' in obj else None rules = _rules(obj['rules'], declared, mins, problems) if 'rules' in obj else () rounds = _rounds(obj['rounds'], declared, problems) if 'rounds' in obj else None + authors = _override_authors(obj.get('override_authors', []), problems) # Rule order: refused in enforce (the rules decide), a warning in shadow (they only say what they would have said). order = _shadowed(rules) warnings = tuple(f'{w}; {SWITCH}' for w in order) if obj.get('mode') == 'shadow' else () @@ -469,4 +478,24 @@ def parse_review(obj, where='review'): if problems: raise ConfigError(f'{where}: ' + '; '.join(problems)) return ReviewRules(obj['version'], obj['mode'], rules, tuple(Lens(n, p) for n, p in lenses.items()), mins, - rounds, obj['findings'], obj['state_line'], warnings, tuple(order)) + rounds, obj['findings'], obj['state_line'], warnings, tuple(order), authors) + + +def _override_authors(raw, problems): + """The override_authors list: GitHub logins, each once (case-insensitive), at most MAX_OVERRIDE_AUTHORS.""" + if not isinstance(raw, list): + problems.append(f'override_authors is {_show(raw)}: must be a list of GitHub logins ([] = no override is a ' + f'person\'s)') + return () + if len(raw) > MAX_OVERRIDE_AUTHORS: + problems.append(f'override_authors has {len(raw)} logins: at most {MAX_OVERRIDE_AUTHORS}') + seen, out = set(), [] + for x in raw: + if not isinstance(x, str) or not LOGIN_RE.fullmatch(x): + problems.append(f'override_authors: {_show(x)} is not a GitHub login') + elif x.lower() in seen: + problems.append(f'override_authors: {x!r} is listed twice') + else: + seen.add(x.lower()) + out.append(x) + return tuple(out) diff --git a/fork-ci/actions/pr-queue/review_judge.py b/fork-ci/actions/pr-queue/review_judge.py index ec49f0b060e5c..403c50b196fd7 100644 --- a/fork-ci/actions/pr-queue/review_judge.py +++ b/fork-ci/actions/pr-queue/review_judge.py @@ -18,16 +18,22 @@ into a failure. judge_rounds judges the review itself, from the lens comments (README.md, "Review rules", Findings and rounds): - - Findings (`findings: true`): a lens whose newest report that counts as a read has a `blocker:` or `major:` line - fails the head, until that lens reads a fixed head in a new comment without one (a later round clears it) or a - person's override comment newer than the report covers it. Owner decision: issue #99. - - Rounds: each different full sha the lens headers name is one fix-and-review round, and a header's round marker - (`r3`) counts as that round too, whichever is higher. From round `rounds.approach_at` the approach lens is - required; a round past `rounds.max` fails the head until a person's override covers every lens: no more rounds, - the person decides (override or close). Owner decision: issue #100. + - Findings (`findings: true`): a lens whose newest entry has a blocker or major finding (read_findings: labels, + headings, bold, tables, folded Unicode; never inside a fenced code block) fails the head, until that lens posts a + new report on a fixed head without one (a later round clears it) or a person's override newer than the report + covers it. A person is a login listed in `override_authors`; none listed (the default) means no override clears + anything. A report with no findings section at all (not even `major: none`) cannot be judged and fails that lens. + Owner decision: issue #99. + - Rounds: a round is a lens read of a different change, not of a different sha: review_round counts the different + own changes (head_check.own_change: the diff from the merge base, so a head that only merges the base in is the + same round) the lens headers name, old, edited and hidden ones too, or the highest round marker (`r3`), whichever + is higher. From round `rounds.approach_at` the approach lens is required; a round past `rounds.max` fails the head + until a person's override for the head, newer than every lens comment, lets it through (it alone is enough: it + also waives the approach lens). Owner decision: issue #100. Pure as judge is: the lens comments come in (lens.Review.lens_scan), the verdict goes out. """ import re +import unicodedata from dataclasses import dataclass import lens @@ -166,52 +172,217 @@ def judge(review, changes, config_path): return Judgement(weight, lenses, hits, tuple(protected), len(paths)) -# A finding line, as a lens report writes it (README.md, "Findings and rounds"): `blocker:` or `major:` at the start -# of a line, after the Markdown lens.norm_line strips (quote and list markers, emphasis, headings), any case, a plural, -# a number and a short note in brackets allowed (`Majors:`, `major 2:`, `major (state):`). Every line counts, in code -# blocks and quotes too: fail closed, as for the lens header. Only an explicit nothing after the colon (NO_FINDING) is -# no finding; any other text is one. -FINDING_RE = re.compile(r'(blocker|major)s?(?:\s*#?\d{1,4})?(?:\s*\([^()]{0,80}\))?\s*:(.*)', re.I) -NO_FINDING_RE = re.compile(r'(?:none|nothing|no|0|n/?a|-+|\u2014|\u2013)\s*\.?', re.I) +# --- findings (issue #99) ----------------------------------------------------------------------------------------- +# What a lens report writes, as the fork's reports are written (README.md, "Findings and rounds"; the table in +# review_judge.test.py, Findings): a label `blocker`, `blocking`, `major` (also plural, `Major findings`, `major 2`, +# `major (state)`), then +# - a separator and the finding on the same line: `major: x`, `**Blocking:** x`, `major — x`, `major - x`, `[major] x`, +# `🔴 Major: x`; or a table row `| major | x |` (or a table whose header row names the label: its column); +# - nothing on the line (`### Major`, `**Blocking**`, `Major:`): the first line under it is the finding. +# Text that says nothing (NONE_RE: `none`, `None found.`, `none (only minors)`, `n/a`, `-` ...) is no finding, and every +# label of a line is judged (`Blocker: none, major: none`). `minor` and `nit` labels never block; they only end the +# text of the label before them. Lines inside a fenced code block hold no label (a report quotes code and test vectors +# there); the line under a bare label is its content even when it opens a fence. A report with no blocker or major +# label at all, not even `major: none`, has no findings section: it cannot be judged (judge_rounds fails it). +# Before any of this the text is folded (fold): NFKC (`Major` -> Major), format characters removed (zero-width space and +# joiner, soft hyphen, bidi marks: category Cf), and Cyrillic and Greek letters that look like Latin ones read as Latin +# (`majоr` with a Cyrillic о); a 0 for an o in a label is read too (`maj0r`). +CONFUSABLES = str.maketrans({ + 'А': 'A', 'В': 'B', 'Е': 'E', 'К': 'K', 'М': 'M', 'Н': 'H', 'О': 'O', 'Р': 'P', 'С': 'C', 'Т': 'T', 'Х': 'X', + 'І': 'I', 'Ј': 'J', 'Ѕ': 'S', 'а': 'a', 'е': 'e', 'о': 'o', 'р': 'p', 'с': 'c', 'х': 'x', 'у': 'y', 'і': 'i', + 'ј': 'j', 'ѕ': 's', 'к': 'k', 'ԁ': 'd', 'ӏ': 'l', 'һ': 'h', 'ԛ': 'q', 'ԝ': 'w', + 'Α': 'A', 'Β': 'B', 'Ε': 'E', 'Ζ': 'Z', 'Η': 'H', 'Ι': 'I', 'Κ': 'K', 'Μ': 'M', 'Ν': 'N', 'Ο': 'O', 'Ρ': 'P', + 'Τ': 'T', 'Υ': 'Y', 'Χ': 'X', 'ο': 'o', 'ι': 'i', 'κ': 'k', 'ν': 'v', 'α': 'a', 'ρ': 'p', + 'ı': 'i', 'ℓ': 'l', 'ǀ': 'l', +}) +_KW = r'(?Pbl[o0]ck(?:ers?|ing)|ma[j][o0]rs?|m[i1]n[o0]rs?|nits?)' +_LABEL = (_KW + r'(?![^\W_])(?:\s+(?:findings?|issues?|problems?|items?|points?|bugs?|ones?)(?![^\W_]))?' + r'(?:\s*#?\d{1,4}(?![^\W_]))?(?:\s*\([^()\n]{0,80}\))?') +_SEP = r'\s*(?::|[—–=]|-(?=\s|$))' +# A label that starts a part of a line, with a separator after it (`Major version: 2` is no label) or in brackets +# (`[major] x`): at the line start, or after punctuation, `and` or `but` (`blocker: none, major: x`). +LABEL_RE = re.compile(r'(?:^|(?<=[,;.!?|/])\s*|\s+(?:and|but)\s+)(?:\[' + _LABEL + r'\]|\(' + _LABEL.replace('kw', 'kw2') + + r'\)|' + _LABEL.replace('kw', 'kw3') + _SEP + ')', re.I) +BARE_RE = re.compile(r'[\[(]?' + _LABEL + r'[\])]?' + _SEP + r'?\s*', re.I) +CELL_RE = re.compile(_LABEL + r'\s*:?\s*', re.I) +NONE_RE = re.compile( + r'(?:none|nothing|nil|no|zero|0|n/?a|-+|—|–)' + r'(?:\s+(?:found|open|left|remaining|seen|here|known|so\s+far|at\s+all|this\s+round|blocking|blockers?|majors?|' + r'findings?|issues?|problems?))*\s*(?:$|[.,;:!?)(\[])', re.I) +FINDINGS_NONE_RE = re.compile(r'(?:findings|issues|problems)\s*:\s*(?:none|nothing|no|0|n/?a)\s*\.?\s*$', re.I) +NOT_SAFE_RE = re.compile(r'\bnot\s+safe\b', re.I) +VERDICT_NOT_SAFE_RE = re.compile(r'verdict\s*[:—–-]?\s*not\s+safe\b', re.I) +RULE_LINE_RE = re.compile(r'\s*(?:[-*_]\s*){3,}') # a thematic break: no content +HEADING_RE = re.compile(r' {0,3}#{1,6}(?:\s|$)') +TABLE_RE = re.compile(r'\s*\|') +TABLE_RULE_RE = re.compile(r'\s*\|?\s*:?-{1,}:?\s*(?:\|\s*:?-{1,}:?\s*)*\|?\s*') ROUND_RE = re.compile(r'r(\d{1,9})') SHOW_FINDING = 120 # characters of a finding line named in the reason +BLOCKING = ('blocker', 'major') -def findings(body): - """[(kind, line)] of the report's blocker and major lines, in order: kind is 'blocker' or 'major', line the line as - normalised (cut to SHOW_FINDING). Read both ways lens reads a line (clean_inline narrow or not): a line either way - reads as a finding is one.""" - out = [] - for raw in (body or '').splitlines(): - for narrow in (False, True): - m = FINDING_RE.match(lens.norm_line(raw, narrow)) - if m and not NO_FINDING_RE.fullmatch(m.group(2).strip()): - out.append((m.group(1).lower(), lens.norm_line(raw, narrow)[:SHOW_FINDING])) +def fold(text): + """The text as the findings rules read it: NFKC, format characters (category Cf) gone, look-alike letters Latin.""" + text = unicodedata.normalize('NFKC', text or '') + return ''.join(ch for ch in text if unicodedata.category(ch) != 'Cf').translate(CONFUSABLES) + + +def _kind(kw): + kw = kw.lower().replace('0', 'o').replace('1', 'i') + return 'blocker' if kw.startswith('block') else 'major' if kw.startswith('major') else 'minor' + + +def _plain(line, narrow): + """One line normalised (lens.norm_line), folded, with leading symbols (emoji, arrows, bullets) gone.""" + return re.sub(r'^[^\w\[(]+', '', lens.norm_line(fold(line), narrow)).strip() + + +def _says_nothing(text): + return bool(NONE_RE.match(re.sub(r'^[^\w\-—–(\[]+', '', text.strip()))) + + +def _cells(raw): + return [c.strip() for c in fold(raw).strip().strip('|').split('|')] + + +def _read(body, narrow): + """(findings [(line index, kind, text)], whether a blocker or major label was seen at all), one way of reading.""" + lines = (body or '').splitlines() + out, seen, fence, columns = [], False, None, None + for i, raw in enumerate(lines): + fence, fenced = lens.fence_step(fence, raw) + if fenced: + columns = None + continue + if TABLE_RE.match(raw): + cells = [_plain(c, narrow) for c in _cells(raw)] + if i + 1 < len(lines) and TABLE_RULE_RE.fullmatch(lines[i + 1]): # a header row: its label columns + columns = {j: _kind(m.group('kw')) for j, c in enumerate(cells) for m in [CELL_RE.fullmatch(c)] if m} + seen |= any(k in BLOCKING for k in columns.values()) + continue + if TABLE_RULE_RE.fullmatch(raw): + continue + for j, c in enumerate(cells): # a label cell: the next cell says what it found + m = CELL_RE.fullmatch(c) + if m and _kind(m.group('kw')) in BLOCKING: + seen = True + said = next((x for x in cells[j + 1:] if x), '') + if not _says_nothing(said): + out.append((i, _kind(m.group('kw')), _plain(raw, narrow))) + for j, kind in (columns or {}).items(): + if kind in BLOCKING and j < len(cells): + if not _says_nothing(cells[j]): + out.append((i, kind, _plain(raw, narrow))) + continue + columns = None + line = _plain(raw, narrow) + if FINDINGS_NONE_RE.match(line): + seen = True + continue + if BARE_RE.fullmatch(line): # `### Major`: the first line under it says what it found + kind = _kind(BARE_RE.fullmatch(line).group('kw')) + if kind not in BLOCKING: + continue + seen = True + under = _under(lines, i + 1, narrow) + if under is None or not _says_nothing(under): + out.append((i, kind, f'{line} (nothing under it)' if under is None else f'{line} {under}')) + continue + marks = list(LABEL_RE.finditer(line)) + for n, m in enumerate(marks): + kind = _kind(m.group('kw') or m.group('kw2') or m.group('kw3')) + if kind not in BLOCKING: + continue + seen = True + said = line[m.end():marks[n + 1].start() if n + 1 < len(marks) else len(line)] + if not _says_nothing(said): + out.append((i, kind, line)) break - return out + return out, seen + + +def _under(lines, start, narrow): + """The first content line after a bare label, normalised, or None when the next one is a heading, another label or + the end (nothing under it). Blank lines and thematic breaks are skipped; a fence line is content.""" + for raw in lines[start:]: + if not raw.strip() or RULE_LINE_RE.fullmatch(raw): + continue + if HEADING_RE.match(raw): + return None + line = _plain(raw, narrow) + if BARE_RE.fullmatch(line) or LABEL_RE.match(line): + return None + return line or raw.strip() + return None + + +def read_findings(body): + """(findings, has_section). findings: [(kind, line)] of the report's blocker and major findings in line order, kind + 'blocker' or 'major', line as normalised (cut to SHOW_FINDING). Read both ways lens reads a line (clean_inline + narrow or not): a line either way reads as a finding is one. has_section: the report has a blocker or major label + at all (`major: none` too), or `findings: none`. A first line (the lens header, or the title line under it) or a + `verdict:` line that says NOT SAFE is a blocker whatever the labels say.""" + found = {} + seen = False + for narrow in (False, True): + out, saw = _read(body, narrow) + seen |= saw + for i, kind, text in out: + found.setdefault(i, (kind, text[:SHOW_FINDING])) + lines = (body or '').splitlines() + top = [i for i, line in enumerate(lines) if line.strip()][:2] + first = set(top if lens.lens_header(body) else top[:1]) # the line after a lens header is a title too + fence = None + for i, raw in enumerate(lines): + fence, fenced = lens.fence_step(fence, raw) + if fenced or i in found: + continue + line = _plain(raw, False) + if (i in first and NOT_SAFE_RE.search(line)) or VERDICT_NOT_SAFE_RE.match(line): + found[i] = ('blocker', f'the report says NOT SAFE: {line}'[:SHOW_FINDING]) + return [found[i] for i in sorted(found)], seen + + +def findings(body): + """[(kind, line)] of the report's blocker and major findings (read_findings).""" + return read_findings(body)[0] def round_marker(name): - """The highest round marker (`r3`) among the words of a lens name after its first, or 0. Emphasis around a word - does not count (`data **r3**`).""" - words = [w.strip('*_`') for w in name.lower().split()[1:]] + """The highest round marker (`r3`) among the words of a lens name after its first, or 0. Markdown emphasis or + backticks around the marker are stripped first (`data **r3**` is round 3), and the name is folded (`r3`).""" + words = [w.strip('*_`') for w in fold(name).lower().split()[1:]] return max((int(m.group(1)) for w in words for m in [ROUND_RE.fullmatch(w)] if m), default=0) -def review_round(history): - """The fix-and-review round the lens headers say: how many different full 40-hex shas they read, or the highest - round marker, whichever is higher. history: [(lens name, sha or None)] of every lens header that counts - (lens.Review.lens_scan), old, edited and hidden ones too. 0: no lens header yet.""" - shas = {sha for _, sha in history if sha and lens.FULL_SHA.fullmatch(sha)} - return max([len(shas)] + [round_marker(name) for name, _ in history]) +def review_round(history, change_of=None): + """The fix-and-review round the lens headers say: how many different changes they read, or the highest round + marker, whichever is higher. history: [(lens name, sha or None)] of every lens header that counts + (lens.Review.lens_scan), old, edited and hidden ones too, a short sha too. change_of(sha): what the pull request + changes at that commit (head_check.own_change: the diff from its merge base with the base, so a commit that only + merges the base in, or another lens's read of the same change, is the same round); without it each different sha + is one. 0: no lens header yet.""" + change_of = change_of or (lambda sha: sha) + changes = {change_of(sha.lower()) for _, sha in history if sha} + return max([len(changes)] + [round_marker(name) for name, _ in history]) + + +# How judge_rounds reads each lens's newest entry (head_check.review_rounds says which one it is): +REPORT = 'report' # a read that counts: its findings, and it needs a findings section +LOOSE = 'loose' # a comment that looks like a report but reads nothing, covered by an override that is no +# person's: its finding lines block, it needs no section (it may be a fix-round note) +UNREAD = 'unread' # a lens report that reads nothing (edited, hidden, a short sha), covered by such an override: +# its findings cannot be read, so it blocks +CLEARED = 'cleared' # a person's override is newer: nothing of it is judged +NOT_JUDGED = 'skip' # it reads nothing and nothing covers it: the head fails for it already @dataclass(frozen=True) class Rounds: round: int # review_round of the lens headers - approach: bool # the approach lens is required (round >= rounds.approach_at) - over_cap: bool # round > rounds.max and no override covers every lens - findings: tuple # (lens key, kind, line) of every blocker or major no later round or override cleared + approach: bool # the approach lens is required (round >= rounds.approach_at and no person decided) + over_cap: bool # round > rounds.max and no person decided + findings: tuple # (lens key, kind, line) of every blocker or major no later round or person cleared problems: tuple # why the head fails, plain text; empty: findings and rounds pass def text(self, review): @@ -225,28 +396,53 @@ def text(self, review): return said + ('; ' + '; '.join(self.problems) if self.problems else '; no blocker or major open') -def judge_rounds(review, history, newest, overridden): +def person_way(review, prefix='override for head'): + """How a person lets a head through, as the reasons say it: who may, and what they write.""" + if not review.override_authors: + return ('a person\'s override comment, but no one is listed in review.override_authors yet, so no override ' + 'is accepted (issue #78: until agents no longer post as the person\'s login, an override is not proof ' + 'of a person)') + return (f'an override comment `{prefix} : ` for this head by ' + f'{" or ".join(review.override_authors)}, newer than every lens comment') + + +def judge_rounds(review, history, reports, decided=False, change_of=None, prefix='override for head'): """Rounds of a pull request's lens comments under the parsed `review` rules. history: [(lens name, sha)] of every - lens header that counts (lens.Review.lens_scan); newest: {lens key: body} of every lens (lens.Review.lens_reads), - the body of its newest report when that report counts as a read, else None (it reads nothing: the head fails for - it already, so it has no findings here); overridden: the lens keys a newer override covers (lens_reads).""" - n, cap = review_round(history), review.rounds.max - every = bool(newest) and set(newest) <= set(overridden) # a person's override covers every lens - over = n > cap and not every - decides = 'a person decides: an override comment for the head, or close the pull request' + lens header that counts (lens.Review.lens_scan); reports: {lens key: (how, body)} of each lens's newest entry, how + one of REPORT, LOOSE, UNREAD, CLEARED, NOT_JUDGED (head_check.review_rounds); decided: a person's override (one of + review.override_authors) for this head is newer than every lens comment; change_of: review_round's. prefix: the + config's override prefix, for the reasons.""" + n, cap = review_round(history, change_of), review.rounds.max + over = n > cap and not decided + way = person_way(review, prefix) problems = [] if over: - problems.append(f'review round {n} is past the cap of {cap} rounds (issue #100): no more rounds; {decides}') + problems.append(f'review round {n} is past the cap of {cap} rounds (issue #100): no more rounds; a person ' + f'decides: close the pull request, or {way}. A person\'s override alone is enough, also for a ' + f'required approach lens that never posted') found = [] if review.findings: - for key in sorted(newest): - if key in overridden or newest[key] is None: + for key in sorted(reports): + how, body = reports[key] + if how in (CLEARED, NOT_JUDGED): + continue + if how == UNREAD: + problems.append(f'lens {key}: its newest report reads nothing and only an override that is no ' + f'person\'s covers it, so its findings cannot be read (issue #99): that lens posts a ' + f'new report, or {way}') continue - for kind, line in findings(newest[key]): + said, section = read_findings(body) + for kind, line in said: found.append((key, kind, line)) + if how == REPORT and not said and not section: + problems.append(f'lens {key}: its newest report has no findings section, so it cannot be judged ' + f'(issue #99): write `blocker: none` and `major: none`, or the findings, as lines of ' + f'the report body (findings in inline review comments are not read)') for key, kind, line in found: - way = (f'this is round {n} of {cap}: no more rounds; {decides}' if n >= cap else - 'fix it and have that lens read the fixed head in a new comment with no blocker or major line, or a ' - 'person posts an override comment for the head') - problems.append(f'lens {key} reported a {kind} that no later round cleared (issue #99): {line!r}; {way}') - return Rounds(n, n >= review.rounds.approach_at, over, tuple(found), tuple(problems)) + again = (f'this is round {n} of {cap}: no more rounds; a person decides: close the pull request, or {way}' + if n >= cap else + f'fix it, and lens {key} posts a new report on the fixed head with no blocker or major ' + f'(`major: none`); or {way}') + problems.append(f'lens {key} reported a {kind} that no later round cleared (issue #99): {line!r}; {again}') + approach = n >= review.rounds.approach_at and not decided + return Rounds(n, approach, over, tuple(found), tuple(problems)) diff --git a/fork-ci/actions/pr-queue/review_judge.test.py b/fork-ci/actions/pr-queue/review_judge.test.py index 0cd2c485c3dac..2f9737d9f15b6 100644 --- a/fork-ci/actions/pr-queue/review_judge.test.py +++ b/fork-ci/actions/pr-queue/review_judge.test.py @@ -165,57 +165,154 @@ def test_the_parser_refuses_those_rule_sets_first(self): # --- findings and rounds (issues #99, #100): judge_rounds on lens comments --------------------------------------- H1, H2, H3, H4, H5 = (c * 40 for c in 'abcde') -BASIC = CONFIGS['basic'] # findings true, rounds approach_at 3, max 4, approach lens `approach` - - -def rounds(history, newest, overridden=(), review=BASIC): - return rj.judge_rounds(review, list(history), dict(newest), set(overridden)) +BASIC = CONFIGS['basic'] # findings true, rounds approach_at 3, max 4, approach lens `approach`, no override_authors +PEOPLE = dataclasses.replace(BASIC, override_authors=('a-person',)) +HEAD = 'lens: code · reviewer: x · read at: ' + H1 + '\n' + + +def rounds(history, reports, decided=False, review=BASIC, change_of=None): + """judge_rounds with each lens's newest entry a report that counts, unless given as (how, body).""" + reports = {k: v if isinstance(v, tuple) else (rj.REPORT, v) for k, v in reports.items()} + return rj.judge_rounds(review, list(history), reports, decided, change_of) + + +# Findings as the fork's lens reports write them (pull requests #720-#739) and as reviews A and B of #740 showed they +# can be written: each must count as a blocker or major finding. +FINDING = [ + ('heading, numbered item', '### Blocking\n\n1. **The lock is gone.** Two writers race on the row.'), + ('heading, bold item', '### Major\n**B1. The retry loops forever.**'), + ('heading with a note', '### Blocking (merge, not safety)\n1. The queue merges an unread head.'), + ('heading with a long note', '### Major (older than this PR; a follow-up issue is fine)\n- the cache never expires'), + ('bold inline', '**Blocking:** the lock is gone.'), + ('bold bare, item under it', '**Major**\n- two writers race'), + ('plain', 'Blocking: x'), + ('findings word', 'Major findings: x'), + ('findings word, list under it', 'Major findings:\n1. x'), + ('dash list', '- major — x'), + ('en dash', 'major – x'), + ('hyphen', 'major - x'), + ('emoji', '🔴 Major: x'), + ('emoji, bold', '⚠️ **Blocker**: x'), + ('numbered label', 'major 2: a retry loops'), + ('plural with a note', 'Majors (state): stopped half-way leaves a row'), + ('brackets', '[major] x'), + ('second label of a line', 'Blocker: none, major: two writers race'), + ('after a semicolon', 'blocker: none; major: x'), + ('after and', 'blocker: none and major: x'), + ('table row', '| severity | finding |\n|---|---|\n| major | x |'), + ('table row, bold', '| **Blocker** | the lock is gone |'), + ('table column', '| blocker | major |\n| --- | --- |\n| none | two writers race |'), + ('fullwidth M (NFKC)', '\uff2dajor: x'), + ('zero-width space', 'maj\u200bor: x'), + ('zero-width joiner in heading', '### Bl\u200docking\n1. x'), + ('soft hyphen', 'ma\u00adjor: x'), + ('Cyrillic o', 'maj\u043er: x'), + ('Cyrillic M', '\u041cajor: x'), + ('Greek omicron', 'bl\u03bfcker: x'), + ('digit zero for o', 'maj0r: x'), + ('heading, code under it', '### Major\n```\nthe race is in x\n```'), + ('heading, nothing under it', '### Major'), + ('heading, next heading under it', '### Major\n### Minor\n- a typo'), + ('bare label, nothing under it', 'major:'), + ('a none that is not one', 'major: none of the retries stop'), + ('a no that is not one', 'major: no lock on the write path'), + ('a dash that is not one', 'major: - x'), + ('see below', 'blocker: see below'), + ('quoted (read: fail closed)', '> major: quoted from round 1'), + ('indented (read: fail closed)', ' major: in an indented block'), + ('HTML tag', 'major: x'), + ('link', '[major](http://e/x): y'), + ('summary tag', '
major: x'), + ('narrow read only', 'major (see '), # the wide tag eats the colon + ('NOT SAFE header', '## Review lens A, head 0123abc: **NOT SAFE**\nblocker: none\nmajor: none'), + ('NOT SAFE verdict', 'verdict: NOT SAFE\nblocker: none\nmajor: none'), +] +# Lines a clean report writes: no finding, and a findings section. +CLEAN = [ + ('both on one line', 'Blocker: none, major: none'), + ('none found', 'Major: none found'), + ('none with a note', 'major: none (only minors)'), + ('none!', 'major: none!'), + ('heading, none under it', '### Major\nnone'), + ('headings, None.', '### Blocking\nNone.\n\n### Major\nNone.'), + ('bold inline', '**Blocking:** none.\n**Major:** none.'), + ('a rule between', '### Major\n\n---\n\nNone.'), + ('table columns', '| blocker | major |\n|---|---|\n| none | none |'), + ('table row', '| severity | finding |\n|---|---|\n| major | none |'), + ('n/a', 'major: n/a'), ('zero', 'major: 0'), ('em dash', 'major: —'), ('list', '- majors: none'), + ('bold no', '**blockers:** no'), ('nothing', 'blocker: nothing'), ('NA', 'major: NA'), + ('minors do not block', 'major: none\n### Minor\n- a typo\nminor: x\nnit: y'), + ('code block', 'major: none\n```\nmajor: in code\n### Blocking\n1. x\n```'), + ('tilde fence', 'blocker: none\n~~~\nblocker: x\n~~~'), + ('longer fence holds a shorter one', 'major: none\n````\n```\nmajor: x\n```\n````'), + ('findings: none', 'findings: none'), + ('Unicode none', '\u041cajor: n\u043ene'), + ('both, full stops', 'Blocker: none. Major: none.'), + ('nothing found', 'major: nothing found'), + ('a minor after it', 'major: none; minor: x'), + ('prose that says not safe', 'major: none\nThe old code was not safe to run twice; this fixes it.'), +] +# Lines that only mention a label: no finding, and no findings section either. +MENTION = ['fixed: the major of round 1', 'no major found', 'majority: x', 'Major version: 2', 'blockerx: y', + 'major 12345: too long a number', 'the blocking call in x'] class Findings(unittest.TestCase): - def test_blocker_and_major_lines_are_findings_minor_ones_are_not(self): - body = ('lens: code · reviewer: x · read at: ' + H1 + '\n' - 'blocker: the lock is gone\n- **Major:** two writers race\nminor: a typo\n### major 2: a retry loops\n' - 'Majors (state): stopped half-way leaves a row\n> major: quoted from round 1\n```\nmajor: in code\n```') - self.assertEqual([k for k, _ in rj.findings(body)], ['blocker', 'major', 'major', 'major', 'major', 'major']) - self.assertEqual(rj.findings(body)[0], ('blocker', 'blocker: the lock is gone')) - - def test_only_an_explicit_nothing_is_no_finding(self): - for line in ('major: none', 'Major: None.', 'blocker: nothing', '**blockers:** no', 'major: 0', 'major: n/a', - 'major: NA', 'major: -', 'major: —', '- majors: none'): - with self.subTest(line): - self.assertEqual(rj.findings(line), []) - for line in ('major:', 'major: no lock on the write path', 'major: none of the retries stop', 'major: - x', - 'blocker: see below'): + def test_each_real_finding_format_counts(self): + for what, body in FINDING: + with self.subTest(what): + said, section = rj.read_findings(HEAD + body) + self.assertTrue(said, body) + self.assertTrue(said[0][0] in ('blocker', 'major')) + + def test_each_clean_format_is_no_finding_and_a_section(self): + for what, body in CLEAN: + with self.subTest(what): + self.assertEqual(rj.read_findings(HEAD + body), ([], True), body) + + def test_a_line_that_only_mentions_a_label_is_neither(self): + for line in MENTION: with self.subTest(line): - self.assertEqual(len(rj.findings(line)), 1) + self.assertEqual(rj.read_findings(HEAD + line), ([], False)) - def test_a_line_that_only_mentions_a_finding_is_none(self): - for line in ('fixed: the major of round 1', 'no major found', 'majority: x', 'Major version: 2', 'blockerx: y', - 'major 12345: too long a number'): - with self.subTest(line): - self.assertEqual(rj.findings(line), []) + def test_the_kind_and_line_are_named(self): + self.assertEqual(rj.findings('blocker: the lock is gone\n- **Major:** two writers race\nminor: a typo'), + [('blocker', 'blocker: the lock is gone'), ('major', 'Major: two writers race')]) + self.assertEqual(rj.findings('### Major\n- race')[0], ('major', 'Major race')) + self.assertEqual(rj.findings('### Major')[0], ('major', 'Major (nothing under it)')) - def test_html_and_links_cannot_hide_a_finding(self): # read both ways, as the lens header is - for line in ('major: x', '[major](http://e/x): y', '
major: x', - 'major (see '): # the last only the narrow read finds: the wide tag eats the colon - with self.subTest(line): - self.assertEqual(len(rj.findings(line)), 1, line) + def test_an_unclosed_fence_runs_to_the_end(self): # GitHub shows the rest as code + self.assertEqual(rj.read_findings('major: none\n```\nmajor: x'), ([], True)) + + def test_fold(self): + self.assertEqual(rj.fold('\uff2da\u200bj\u043er'), 'Major') + self.assertEqual(rj.fold(None), '') class Rounds(unittest.TestCase): - def test_each_different_full_sha_read_is_a_round(self): + def test_each_different_change_read_is_a_round(self): self.assertEqual(rj.review_round([]), 0) self.assertEqual(rj.review_round([('code', H1), ('data-1', H1)]), 1) self.assertEqual(rj.review_round([('code', H1), ('code', H2), ('data-1', H2), ('code', H3)]), 3) - self.assertEqual(rj.review_round([('code', 'abcdef1'), ('code', None)]), 0) # a short sha or none: no round + self.assertEqual(rj.review_round([('code', None)]), 0) + + def test_a_short_sha_counts_too(self): # it reads nothing for the head, but it was a read: a round + self.assertEqual(rj.review_round([('code', 'abcdef1'), ('code', H1)]), 2) + + def test_reads_of_one_change_are_one_round(self): # #100: a queue update merges the base in, no fix + change = {H1: 'x', H2: 'x', H3: 'y'}.get + self.assertEqual(rj.review_round([('code', H1), ('code', H2)], change), 1) + self.assertEqual(rj.review_round([('code', H1), ('code', H2), ('code', H3)], change), 2) + self.assertEqual(rj.review_round([('code', H1.upper()), ('code', H1)]), 1) def test_a_round_marker_counts_when_it_is_higher(self): # deleted old rounds: the marker still says it self.assertEqual(rj.review_round([('code r3', H1)]), 3) - self.assertEqual(rj.review_round([('code **r4**', H1), ('data r2', H2)]), 4) + self.assertEqual(rj.review_round([('code **r4**', H1), ('data r2', H2)]), 4) # emphasis stripped: counts + self.assertEqual(rj.review_round([('code `r5`', H1)]), 5) self.assertEqual(rj.review_round([('code r1', H1), ('code r1', H2), ('code', H3)]), 3) self.assertEqual(rj.round_marker('r2d2 lens'), 0) self.assertEqual(rj.round_marker('r9'), 0) # a name's first word is its name, never a marker + self.assertEqual(rj.round_marker('code r\uff13'), 3) # folded def test_no_lens_comment_yet_is_round_zero_and_passes(self): # empty state r = rounds([], {}) @@ -225,19 +322,30 @@ def test_a_major_blocks_until_a_later_round_of_that_lens_clears_it(self): r = rounds([('code', H1)], {'code': 'major: two writers race'}) self.assertEqual(r.findings, (('code', 'major', 'major: two writers race'),)) self.assertIn('lens code reported a major that no later round cleared', r.problems[0]) - self.assertIn('read the fixed head in a new comment', r.problems[0]) + self.assertIn('lens code posts a new report on the fixed head', r.problems[0]) r = rounds([('code', H1), ('code r2', H2)], {'code': 'major: none'}) # round 2 cleared it self.assertEqual((r.round, r.problems), (2, ())) - def test_an_override_newer_than_the_report_clears_its_findings(self): # #99: only a person's override - r = rounds([('code', H1), ('data-1', H1)], {'code': 'blocker: x', 'data-1': 'major: y'}, overridden={'code'}) - self.assertEqual([f[0] for f in r.findings], ['data-1']) - r = rounds([('code', H1)], {'code': 'blocker: x'}, overridden={'code'}) - self.assertEqual(r.problems, ()) + def test_a_report_with_no_findings_section_fails_that_lens(self): # #99: never passes as clean + r = rounds([('code', H1)], {'code': HEAD + 'Looks good to me.'}) + self.assertEqual(len(r.problems), 1) + self.assertIn('lens code: its newest report has no findings section', r.problems[0]) + self.assertIn('findings in inline review comments are not read', r.problems[0]) + self.assertEqual(rounds([('code', H1)], {'code': HEAD}).problems, r.problems) # empty body: the same + self.assertEqual(rounds([('code', H1)], {'code': HEAD + 'major: none'}).problems, ()) - def test_a_lens_that_reads_nothing_is_not_judged_here(self): # it fails the head already (edited, hidden) - r = rounds([('code', H1)], {'code': None}) - self.assertEqual(r.problems, ()) + def test_an_entry_that_reads_nothing_under_an_agents_override_cannot_be_judged(self): + r = rounds([('code', H1)], {'code': (rj.UNREAD, HEAD + 'major: none')}) + self.assertIn('its findings cannot be read', r.problems[0]) + + def test_a_loose_comment_under_an_agents_override_is_judged_by_its_lines(self): # no section needed + self.assertEqual(rounds([], {'x': (rj.LOOSE, 'fixed it')}).problems, ()) + self.assertEqual(len(rounds([], {'x': (rj.LOOSE, 'major: still racy')}).findings), 1) + + def test_a_cleared_or_unjudged_entry_is_not_judged_here(self): + for how in (rj.CLEARED, rj.NOT_JUDGED): # a person's override; or it fails the head already + with self.subTest(how): + self.assertEqual(rounds([('code', H1)], {'code': (how, 'blocker: x')}).problems, ()) def test_findings_false_judges_no_finding(self): r = rounds([('code', H1)], {'code': 'blocker: x'}, review=dataclasses.replace(BASIC, findings=False)) @@ -245,36 +353,59 @@ def test_findings_false_judges_no_finding(self): self.assertIn('findings are not judged', r.text(dataclasses.replace(BASIC, findings=False))) def test_the_approach_lens_is_required_from_round_approach_at(self): - self.assertFalse(rounds([('code', H1), ('code', H2)], {'code': ''}).approach) - r = rounds([('code', H1), ('code', H2), ('code', H3)], {'code': ''}) + self.assertFalse(rounds([('code', H1), ('code', H2)], {'code': 'major: none'}).approach) + r = rounds([('code', H1), ('code', H2), ('code', H3)], {'code': 'major: none'}) self.assertTrue(r.approach) self.assertIn('approach lens approach required from round 3', r.text(BASIC)) + self.assertFalse(rounds([('code', H1), ('code', H2), ('code', H3)], {}, decided=True).approach) - def test_a_round_past_the_cap_stops_and_a_person_decides(self): # #100 + def test_a_round_past_the_cap_stops_and_says_what_a_person_does(self): # #100 four = [('code', h) for h in (H1, H2, H3, H4)] - self.assertEqual(rounds(four, {'code': ''}).problems, ()) # round 4 of 4 may pass - r = rounds(four + [('code', H5)], {'code': ''}) - self.assertTrue(r.over_cap) - self.assertEqual(r.problems, ('review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a ' - 'person decides: an override comment for the head, or close the pull request',)) - self.assertEqual(rounds([('code r5', H1)], {'code': ''}).round, 5) - - def test_an_override_covering_every_lens_passes_the_cap_a_newer_round_does_not(self): - five = [('code', h) for h in (H1, H2, H3, H4, H5)] + [('approach', H5)] - self.assertEqual(rounds(five, {'code': '', 'approach': ''}, overridden={'code', 'approach'}).problems, ()) - r = rounds(five, {'code': '', 'approach': ''}, overridden={'code'}) # approach posted after the override + self.assertEqual(rounds(four, {'code': 'major: none'}).problems, ()) # round 4 of 4 may pass + r = rounds(four + [('code', H5)], {'code': 'major: none'}) self.assertTrue(r.over_cap) + self.assertEqual(r.problems, ( + 'review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a person decides: close the pull ' + 'request, or a person\'s override comment, but no one is listed in review.override_authors yet, so no ' + 'override is accepted (issue #78: until agents no longer post as the person\'s login, an override is not ' + 'proof of a person). A person\'s override alone is enough, also for a required approach lens that never ' + 'posted',)) + r = rounds(four + [('code', H5)], {'code': 'major: none'}, review=PEOPLE) + self.assertIn('an override comment `override for head : ` for this head by a-person, newer ' + 'than every lens comment', r.problems[0]) + self.assertEqual(rounds([('code r5', H1)], {'code': 'major: none'}).round, 5) + + def test_a_persons_decision_passes_the_cap_and_waives_the_approach_lens(self): + five = [('code', h) for h in (H1, H2, H3, H4, H5)] + r = rounds(five, {'code': (rj.CLEARED, '')}, decided=True, review=PEOPLE) + self.assertEqual((r.over_cap, r.approach, r.problems), (False, False, ())) def test_a_finding_at_the_last_round_says_no_more_rounds(self): r = rounds([('code', h) for h in (H1, H2, H3, H4)], {'code': 'major: still racy'}) self.assertIn('this is round 4 of 4: no more rounds; a person decides', r.problems[0]) def test_the_same_comments_give_the_same_verdict(self): # runs again: pure - args = ([('code', H1), ('code r2', H2)], {'code': 'major: x'}, {'data-1'}) + args = ([('code', H1), ('code r2', H2)], {'code': 'major: x'}) self.assertEqual(rounds(*args), rounds(*args)) -# --- the fork's own config (issue #730): in enforce, workflow, queue and lens files critical (#82) ---------------- +class OverrideAuthors(unittest.TestCase): + def test_left_out_or_empty_is_no_person(self): # #78: agents still post as the owner's login + self.assertEqual(BASIC.override_authors, ()) + block = VECTORS['configs']['basic'] + self.assertEqual(rc.parse_review({**block, 'override_authors': []}).override_authors, ()) + self.assertEqual(rc.parse_review({**block, 'override_authors': ['a-person', 'B2']}).override_authors, + ('a-person', 'B2')) + + def test_a_wrong_list_is_refused(self): + block = VECTORS['configs']['basic'] + for bad in ('a-person', [''], ['-x'], ['a b'], ['x', 'X'], [1], ['x'] * 21, None): + with self.subTest(bad), self.assertRaises(config.ConfigError) as e: + rc.parse_review({**block, 'override_authors': bad}) + self.assertIn('override_authors', str(e.exception)) + + +# --- the fork's own config (issue #730): in shadow, workflow, queue and lens files critical (#82) ---------------- ROOT = os.path.join(HERE, '..', '..', '..') FORK_CONFIG = '.github/fork-pr-queue.json' @@ -285,10 +416,11 @@ def setUp(self): self.cfg = config.parse(f.read(), FORK_CONFIG) self.review = self.cfg.review - def test_it_parses_in_enforce_with_the_owners_numbers(self): - r = self.review - self.assertEqual((r.mode, r.findings, r.rounds.approach_at, r.rounds.max, r.warnings), - ('enforce', True, 3, 4, ())) + def test_it_parses_in_shadow_with_the_owners_numbers_and_no_person_yet(self): + r = self.review # shadow first (README, Turning it on); no override author until #78 + self.assertEqual((r.mode, r.findings, r.rounds.approach_at, r.rounds.max, r.warnings, r.override_authors), + ('shadow', True, 3, 4, (), ())) + self.assertEqual(r.order, ()) # enforce would take it as it is self.assertGreaterEqual(r.min_lenses.critical, r.min_lenses.normal + 1) # #82: one extra review for x in r.lenses: with self.subTest(x.name): From ab4c4fc9520bef5843c853b83ca7aadf7b37af3c Mon Sep 17 00:00:00 2001 From: John Doe <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 16:17:35 +0100 Subject: [PATCH 3/3] fork ci: a findings section needs both labels, read finding: lines, key rounds on content (#740 final fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A report has a findings section only with a blocker label and a major label (or findings: none, or finding: lines): blocker: none plus a major in a form not read (F1 (major): x, Severity: major, One major: x ...) now fails closed instead of passing as clean. - finding: lines are read: a minor-only report passes, a finding: major line fails whatever none lines are there. - Clean: major: none — two minors below, Major: (none), none plus an emoji. A labelled recap (Major 1: fixed) stays an open finding; the lens prompts say to write recaps without a label. - HTML entities, backslash escapes and the ratio sign are folded. - Linear time on long runs of spaces (squeezed in _plain; the table rule pattern matched on the stripped line): a 200k-character line under 2 s. - Round key: index lines and the whole hunk header dropped, --binary kept, so a base edit to another line of a file the PR changes is no new round. - A listed override login whose comment has another author association is warned about in the check output; a read-commit fetch has a timeout. - Tests for the hunk-header strip, the fence that ends a table and the override_authors cap. Co-Authored-By: Claude Opus 5.5 --- .github/fork-lens-approach.md | 2 + .github/fork-lens-code.md | 2 + .github/fork-lens-merge-path.md | 2 + .github/fork-lens-state.md | 2 + fork-ci/actions/head-check/README.md | 2 +- fork-ci/actions/head-check/head_check.py | 33 +++++-- fork-ci/actions/head-check/head_check.test.py | 64 +++++++++++++- fork-ci/actions/pr-queue/README.md | 42 ++++++--- fork-ci/actions/pr-queue/review_judge.py | 66 ++++++++------ fork-ci/actions/pr-queue/review_judge.test.py | 88 ++++++++++++++++--- 10 files changed, 239 insertions(+), 64 deletions(-) diff --git a/.github/fork-lens-approach.md b/.github/fork-lens-approach.md index cc7ed4d990ef6..af3f4556efa96 100644 --- a/.github/fork-lens-approach.md +++ b/.github/fork-lens-approach.md @@ -19,6 +19,8 @@ Then the findings section, always, also when it is empty: or, for each finding, one line that starts `blocker:` or `major:` (`minor:` for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. A report without its `blocker:` and `major:` lines cannot be judged and fails the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings and rounds"). diff --git a/.github/fork-lens-code.md b/.github/fork-lens-code.md index 6f66fc3d8ca81..3d4c47ad682a6 100644 --- a/.github/fork-lens-code.md +++ b/.github/fork-lens-code.md @@ -20,6 +20,8 @@ Then the findings section, always, also when it is empty: or, for each finding, one line that starts `blocker:` or `major:` (`minor:` for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. A report without its `blocker:` and `major:` lines cannot be judged and fails the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings and rounds"). diff --git a/.github/fork-lens-merge-path.md b/.github/fork-lens-merge-path.md index 2f0992994eaf7..436e0ea0cfed5 100644 --- a/.github/fork-lens-merge-path.md +++ b/.github/fork-lens-merge-path.md @@ -24,6 +24,8 @@ Then the findings section, always, also when it is empty: or, for each finding, one line that starts `blocker:` or `major:` (`minor:` for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. A report without its `blocker:` and `major:` lines cannot be judged and fails the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings and rounds"). diff --git a/.github/fork-lens-state.md b/.github/fork-lens-state.md index c7867fa163675..27624dd284132 100644 --- a/.github/fork-lens-state.md +++ b/.github/fork-lens-state.md @@ -21,6 +21,8 @@ Then the findings section, always, also when it is empty: or, for each finding, one line that starts `blocker:` or `major:` (`minor:` for the rest). Quote code or example findings only inside a fenced code block. +Write a finding of an earlier round without a label (`round 1, fixed: x`): a +`blocker:` or `major:` line is read as open, also `Major 1: fixed`. A report without its `blocker:` and `major:` lines cannot be judged and fails the head (fork-ci/actions/pr-queue/README.md, "Lens comments" and "Findings and rounds"). diff --git a/fork-ci/actions/head-check/README.md b/fork-ci/actions/head-check/README.md index 3571169aa8b76..1416c13a184f4 100644 --- a/fork-ci/actions/head-check/README.md +++ b/fork-ci/actions/head-check/README.md @@ -56,7 +56,7 @@ All of them are the queue's, imported from `../pr-queue`, never copied: a `### Blocking` or `### Major` heading with items, a table row ...) fails the head until a later round of that lens clears it or a person's override newer than the report covers it (`findings: true`); a report with no findings - section at all fails too. A person is a login in `override_authors`; none + section (a blocker label and a major label, `none` allowed) fails too. A person is a login in `override_authors`; none listed (the default) means no override clears a finding or passes the cap (issue #78). A round is a read of a different own change (`own_change`: a head that only merges the base in is no new round); from round diff --git a/fork-ci/actions/head-check/head_check.py b/fork-ci/actions/head-check/head_check.py index 8457260fcabe1..03adf131c7704 100644 --- a/fork-ci/actions/head-check/head_check.py +++ b/fork-ci/actions/head-check/head_check.py @@ -55,6 +55,7 @@ import review_judge # noqa: E402 ATTEMPTS, RETRY_SECONDS = 3, 10 +FETCH_SECONDS = 120 # a fetch of a read commit that takes longer counts as failed (the job has 15 minutes in all) FIELDS = 'isDraft,headRefOid,comments,reviews' SHA = pr_queue.SHA REFUSED = pr_queue.REFUSED @@ -125,10 +126,14 @@ def has(self, sha): def fetch(self, sha): """A read commit no ref of the repository reaches (a force-push dropped it; GitHub keeps it), from origin with the - credentials the checkout stored or the gate's environment gives (none: a private repository then fails).""" - return self.git('fetch', '-q', '--no-tags', '--no-recurse-submodules', self.remote, sha, - fetch=True).returncode == 0 \ - and self.has(sha) + credentials the checkout stored or the gate's environment gives (none: a private repository then fails). A fetch + that takes over FETCH_SECONDS failed.""" + try: + done = self.git('fetch', '-q', '--no-tags', '--no-recurse-submodules', self.remote, sha, fetch=True, + timeout=FETCH_SECONDS).returncode == 0 + except subprocess.TimeoutExpired: + return False + return done and self.has(sha) def required_names(text): @@ -375,9 +380,11 @@ def own_change(checkout, base): """change_of for review_judge.review_round: what the pull request itself changes at a read commit, so two reads of the same change are one round. The key is a hash of `git diff -U0` from the commit's merge base with `base` to it, hunk line numbers dropped: a head that only merges the base in (a queue or branch update) has the same own change - as the head it updates, and is no new round. A short sha is resolved when it names one commit. A commit that is not - in the repository and cannot be fetched, more than one merge base or a failed diff: a round of its own (unknown - counts more rounds, never fewer).""" + as the head it updates, and is no new round. The `index ..` lines and the whole hunk header (line + numbers, and the function name git shows) are dropped too: they move when the base edits other lines of a file the + pull request changes; --binary keeps a binary file's content in the key. A short sha is resolved when it names one + commit. A commit that is not in the repository and cannot be fetched, more than one merge base or a failed diff: a + round of its own (unknown counts more rounds, never fewer).""" cache = {} def of(sha): @@ -402,11 +409,11 @@ def diff_key(checkout, base, sha): if len(bases) != 1 or not SHA.fullmatch(bases[0]): return None d = checkout.git('diff', '--no-ext-diff', '--no-textconv', '--no-relative', '--ignore-submodules=none', - '--no-color', '--no-renames', '--full-index', '-U0', bases[0], full, '--', + '--no-color', '--no-renames', '--binary', '-U0', bases[0], full, '--', errors='surrogateescape') if d.returncode != 0: return None - text = re.sub(r'^@@ [^@]* @@', '@@', d.stdout, flags=re.M) + text = re.sub(r'^index .*\n', '', re.sub(r'^@@ .*', '@@', d.stdout, flags=re.M), flags=re.M) return ('change', hashlib.sha256(text.encode('utf-8', 'surrogateescape')).hexdigest()) @@ -425,6 +432,14 @@ def review_rounds(rules, review, pr, checkout, prefix): return None, "HEAD is not GitHub's two-parent merge commit, so the rounds are unknown" entries, history = review.lens_scan(pr) people = {a.lower() for a in rules.override_authors} + for _, _, post in review.lens_posts(pr): # listed, but GitHub records another association: it never counts + login = (post.get('author') or {}).get('login') or '' + if login.lower() in people and review.override_re.match(post.get('body') or '') \ + and post.get('authorAssociation') != review.override_association: + review.log(f'warning: the override comment by {login} does not count: GitHub records its author ' + f'association as {post.get("authorAssociation") or "none"}, and an override counts only ' + f'with {review.override_association} (override.author_association); being listed in ' + f'review.override_authors does not change that') overrides = [((when, i), sha, post) for when, i, post in review.lens_posts(pr) for sha in [review.override_sha(post)] if sha] newest = max((at for at, _, _ in overrides), default=None) diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index 5132fa0a6b2bd..a55c12804d0cc 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -1313,7 +1313,7 @@ def test_shadow_only_says_what_it_would_have_said(self): def test_a_later_round_with_no_finding_clears_it(self): # runs again on the same pull request after a fix base, head = self.pair() old = self.said('code', OLD[0], 'major: two writers race') - code, out = self.check(base, head, comments=[old, self.said('code r2', head, 'major: none')]) + code, out = self.check(base, head, comments=[old, self.said('code r2', head)]) self.assertEqual(code, 0, out) self.assertIn('findings and rounds (enforce): round 2 of 4; no blocker or major open', out) @@ -1325,7 +1325,7 @@ def test_a_round_stopped_half_way_keeps_the_old_finding(self): # the fix was p def test_a_deleted_newer_report_brings_the_old_finding_back(self): # deleted later: GitHub no longer lists it base, head = self.pair() - first, second = self.said('code', head, 'blocker: x'), self.said('code r2', head, 'blocker: none') + first, second = self.said('code', head, 'blocker: x'), self.said('code r2', head) self.assertEqual(self.check(base, head, comments=[first, second])[0], 0) code, out = self.check(base, head, comments=[first]) self.assertEqual(code, 1, out) @@ -1405,7 +1405,12 @@ def test_a_report_with_no_findings_section_fails_the_head(self): # #99: never code, out = self.check(base, head, comments=[self.said('code', head, 'Looks good to me.')]) self.assertEqual(code, 1, out) self.assertIn('lens code: its newest report has no findings section, so it cannot be judged (issue #99)', out) - self.assertEqual(self.check(base, head, comments=[self.said('code', head, '### Major', 'None.')])[0], 0) + self.assertEqual(self.check(base, head, comments=[self.said('code', head, '### Major', 'None.')])[0], 1) + code, out = self.check(base, head, comments=[self.said('code', head, 'blocker: none', 'F1 (major): x')]) + self.assertEqual(code, 1, out) # #740 round 2: one label kind and a major in a form not read: no section + self.assertIn('lens code: its newest report has no findings section', out) + clean = ('### Blocking', 'None.', '### Major', 'None.') + self.assertEqual(self.check(base, head, comments=[self.said('code', head, *clean)])[0], 0) def test_a_finding_in_a_heading_or_table_or_lookalike_fails_the_head(self): # #99, both ways base, head = self.pair() @@ -1436,6 +1441,47 @@ def test_a_queue_update_is_no_new_round(self): # #100: a round is a read after self.assertEqual(code, 1, out) self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + def test_a_base_edit_to_a_file_the_pull_request_changes_is_no_new_round(self): # #740 round 2, B major 1 + cfg = review_cfg('enforce') + app = ['top', 'mid', '.', '.', ' 1', ' 2', '.', 'end'] + + def tree(*lines): + return base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.py': '\n'.join(lines) + '\n'}) + base = self.w.commit(tree(*app), [self.m0]) + h1 = self.w.commit(tree(*app[:7], 'END'), [base]) + h2 = self.w.commit(tree(*app[:4], ' 1x', *app[5:7], 'END'), [h1]) # the fix: round 2 + base2 = self.w.commit(tree('new', 'top', 'MID', *app[2:]), [base]) # other lines of the same file + h3 = self.w.merge(h2, base2) # line numbers, `index` blobs and the hunk's function name (MID) all move + reads = [self.said('code', h1), self.said('code r2', h2), self.said('code', h3)] + code, out = self.check(base2, h3, comments=reads) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4; no blocker or major open', out) + fixed = self.w.commit(tree('new', 'top', 'MID', *app[2:4], ' 1y', *app[5:7], 'END'), [h3]) # its own change + code, out = self.check(base2, fixed, comments=reads + [self.said('code', fixed)]) + self.assertEqual(code, 1, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + + def test_a_binary_file_changed_again_is_a_new_round(self): # --binary: its content is in the key + base, _ = self.pair() + cfg = review_cfg('enforce') + h1, h2 = (self.w.commit(base_files(cfg_raw=cfg, **{**PROMPTS, **self.STORE, 'src/app.bin': blob}), [base]) + for blob in ('\0one\n', '\0two\n')) + code, out = self.check(base, h2, comments=[self.said('code', h1), self.said('code', h2)]) + self.assertEqual(code, 0, out) + self.assertIn('round 2 of 4', out) + + def test_a_listed_login_without_the_override_association_is_warned_about(self): # #740 round 2, B minor 3 + base, head = self.pair(override_authors=['a-collaborator', 'owner']) + report = self.said('code', head, 'blocker: x', assoc='COLLABORATOR', login='a-collaborator') # no override + listed = self.p.override(head, assoc='COLLABORATOR', login='a-collaborator') + code, out = self.check(base, head, comments=[report, listed, self.p.override(OLD[0])]) # owner's: counts + self.assertEqual(code, 1, out) + self.assertIn('warning: the override comment by a-collaborator does not count: GitHub records its author ' + 'association as COLLABORATOR, and an override counts only with OWNER', out) + self.assertEqual(out.count('warning: the override comment'), 1, out) + other = self.p.override(head, assoc='COLLABORATOR', login='someone') # not listed: no warning + self.assertNotIn('warning: the override comment', self.check(base, head, comments=[report, other])[1]) + def test_a_short_sha_read_still_counts_a_round(self): # old state: a lens that wrote 12 characters base, head = self.pair() short = self.said('code', OLD[0][:12]) @@ -1492,6 +1538,18 @@ class FindingsAndRoundsBare(FindingsAndRounds): BARE = True +class FetchTimeout(unittest.TestCase): # #740 round 2, B minor 4: a hung fetch of a read commit never hangs the check + def test_a_fetch_that_takes_too_long_failed(self): + timeouts = [] + + def run(cmd, **kw): + timeouts.append(kw.get('timeout')) + raise subprocess.TimeoutExpired(cmd, kw.get('timeout')) + with tempfile.TemporaryDirectory() as d, unittest.mock.patch.object(H.subprocess, 'run', run): + self.assertFalse(H.Checkout(d, bare=(A, A)).fetch(A)) + self.assertEqual(timeouts, [H.FETCH_SECONDS]) + + class GitEnvironment(unittest.TestCase): # #442 item 2: pr_queue.GIT_RUN """The gate's git calls get no program or transport of the job's environment where none can be needed: its local calls reach no remote, and its fetch from GitHub (https) needs git's own helpers only. The checkout mode fetches diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index 9a0fd852cf1af..7ff83d13675d4 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -205,21 +205,35 @@ Owner decisions: issue #99 (findings), issue #100 (rounds). `| major | x |` or a table column headed `major`), or, for a bare label, the first line under it (a heading or nothing under it is a finding too). Every label of a line counts (`blocker: none, major: x`). Only an explicit nothing - is no finding (`none`, `None found`, `none (only minors)`, `nothing`, `no`, - `0`, `n/a`, `-`). Before reading, the text is folded: NFKC, zero-width and - other format characters removed, Cyrillic and Greek look-alike letters read - as Latin, `0` for `o` in a label. Lines inside a fenced code block are never - labels, so quote code and example findings in one; quotes and indented - blocks are read (fail closed). A title line or `verdict:` line that says + is no finding (`none`, `None found`, `none (only minors)`, `(none)`, + `none — two minors below`, `nothing`, `no`, `0`, `n/a`, `-`). A recap of an + earlier round with a label (`Major 1: fixed`) is read as an open finding: + write it without one (`round 1, fixed: x`). The format of one line per + finding, `finding: ...`, is read too (with + `findings: none` when there is none); only its minors pass. Before reading, + the text is folded: NFKC, zero-width and other format characters removed, + Cyrillic and Greek look-alike letters read as Latin, `0` for `o` in a label, + HTML entities decoded (`major`) and backslash escapes dropped + (`major\:`). Lines inside a fenced code block are never labels, so quote + code and example findings in one; quotes and indented blocks are read (fail + closed). A title line or `verdict:` line that says NOT SAFE is a blocker. `minor:` and `nit:` never block. A lens whose newest report has a blocker or major fails the head. A report - with no findings section at all (not even `major: none`) cannot be judged and - fails that lens. Findings go in the report body: findings in inline review - comments (on a line of the diff) are not read, so a lens posted as a PR review - whose body has no findings section fails. The ways out: fix it, and the lens + without a findings section cannot be judged and fails that lens: it needs a + blocker label and a major label (`blocker: none` and `major: none` too), or + `findings: none`, or `finding:` lines. One of the two labels is not enough: + `blocker: none` with the major written another way (`F1 (major): x`) would + read as clean. A report that writes `major: none` and then a major in a form + not read here contradicts itself, and is read by its `major: none`. Findings + go in the report body: findings in inline review comments (on a line of the + diff) are not read, so a lens posted as a PR review whose body has no + findings section fails. The ways out: fix it, and the lens posts a new report on the fixed head without a blocker or major; or a person posts an override comment for the head (config `override`) newer than the - report. A person is a login listed in `override_authors`; an override by any + report. A person is a login listed in `override_authors` whose override + comment GitHub also records with the override association + (`override.author_association`); an override by a listed login with another + association does not count, and the check output warns about it. An override by any other author of the override association covers a lens's read as before, but clears no finding and passes no cap. While agents still post as the owner's login (issue #78) an override from it proves no person, so keep the list @@ -227,8 +241,10 @@ Owner decisions: issue #99 (findings), issue #100 (rounds). finding is fixed or the pull request closed. - **Rounds:** a round is a lens read of a fixed change, not of a new sha. The gate counts the different own changes the counted lens headers read: for each - read sha, `git diff -U0` from its merge base with the base to it, line numbers - dropped (`head_check.own_change`). A head that only merges the base in (a + read sha, `git diff -U0 --binary` from its merge base with the base to it, + the hunk headers (line numbers, function names) and `index` lines dropped + (`head_check.own_change`), so a base that edits other lines of a file the + pull request changes moves nothing. A head that only merges the base in (a queue update, `gh pr update-branch`) or the same change rebased onto a newer base is the same change, so the same round; a merge that also changes the pull request's own diff is a new one. A short sha counts as well, and a sha the diff --git a/fork-ci/actions/pr-queue/review_judge.py b/fork-ci/actions/pr-queue/review_judge.py index 403c50b196fd7..10e8e79273298 100644 --- a/fork-ci/actions/pr-queue/review_judge.py +++ b/fork-ci/actions/pr-queue/review_judge.py @@ -22,7 +22,8 @@ headings, bold, tables, folded Unicode; never inside a fenced code block) fails the head, until that lens posts a new report on a fixed head without one (a later round clears it) or a person's override newer than the report covers it. A person is a login listed in `override_authors`; none listed (the default) means no override clears - anything. A report with no findings section at all (not even `major: none`) cannot be judged and fails that lens. + anything. A report with no findings section (a blocker label and a major label, `none` allowed) cannot be judged + and fails that lens. Owner decision: issue #99. - Rounds: a round is a lens read of a different change, not of a different sha: review_round counts the different own changes (head_check.own_change: the diff from the merge base, so a head that only merges the base in is the @@ -32,6 +33,7 @@ also waives the approach lens). Owner decision: issue #100. Pure as judge is: the lens comments come in (lens.Review.lens_scan), the verdict goes out. """ +import html import re import unicodedata from dataclasses import dataclass @@ -182,18 +184,23 @@ def judge(review, changes, config_path): # Text that says nothing (NONE_RE: `none`, `None found.`, `none (only minors)`, `n/a`, `-` ...) is no finding, and every # label of a line is judged (`Blocker: none, major: none`). `minor` and `nit` labels never block; they only end the # text of the label before them. Lines inside a fenced code block hold no label (a report quotes code and test vectors -# there); the line under a bare label is its content even when it opens a fence. A report with no blocker or major -# label at all, not even `major: none`, has no findings section: it cannot be judged (judge_rounds fails it). +# there); the line under a bare label is its content even when it opens a fence. A report has a findings section only +# when it has a blocker label AND a major label (`blocker: none` and `major: none` too), or `findings: none`, or +# `finding: ...` lines: with one of the two labels only, a major written in a form not read here +# (`F1 (major): x`) would pass as clean, so such a report cannot be judged (judge_rounds fails it). Each finding is one +# line `finding: ...` in that format (FINDING_LINE_RE). A recap of an earlier round with a label +# (`Major 1: fixed`) is read as a finding like any other; write it without one (`round 1, fixed: x`). # Before any of this the text is folded (fold): NFKC (`Major` -> Major), format characters removed (zero-width space and # joiner, soft hyphen, bidi marks: category Cf), and Cyrillic and Greek letters that look like Latin ones read as Latin -# (`majоr` with a Cyrillic о); a 0 for an o in a label is read too (`maj0r`). +# (`majоr` with a Cyrillic о); a 0 for an o in a label is read too (`maj0r`); HTML entities decoded (`major`) and +# Markdown backslash escapes dropped (`major\: x`), as GitHub renders them. CONFUSABLES = str.maketrans({ 'А': 'A', 'В': 'B', 'Е': 'E', 'К': 'K', 'М': 'M', 'Н': 'H', 'О': 'O', 'Р': 'P', 'С': 'C', 'Т': 'T', 'Х': 'X', 'І': 'I', 'Ј': 'J', 'Ѕ': 'S', 'а': 'a', 'е': 'e', 'о': 'o', 'р': 'p', 'с': 'c', 'х': 'x', 'у': 'y', 'і': 'i', 'ј': 'j', 'ѕ': 's', 'к': 'k', 'ԁ': 'd', 'ӏ': 'l', 'һ': 'h', 'ԛ': 'q', 'ԝ': 'w', 'Α': 'A', 'Β': 'B', 'Ε': 'E', 'Ζ': 'Z', 'Η': 'H', 'Ι': 'I', 'Κ': 'K', 'Μ': 'M', 'Ν': 'N', 'Ο': 'O', 'Ρ': 'P', 'Τ': 'T', 'Υ': 'Y', 'Χ': 'X', 'ο': 'o', 'ι': 'i', 'κ': 'k', 'ν': 'v', 'α': 'a', 'ρ': 'p', - 'ı': 'i', 'ℓ': 'l', 'ǀ': 'l', + 'ı': 'i', 'ℓ': 'l', 'ǀ': 'l', '∶': ':', }) _KW = r'(?Pbl[o0]ck(?:ers?|ing)|ma[j][o0]rs?|m[i1]n[o0]rs?|nits?)' _LABEL = (_KW + r'(?![^\W_])(?:\s+(?:findings?|issues?|problems?|items?|points?|bugs?|ones?)(?![^\W_]))?' @@ -206,16 +213,17 @@ def judge(review, changes, config_path): BARE_RE = re.compile(r'[\[(]?' + _LABEL + r'[\])]?' + _SEP + r'?\s*', re.I) CELL_RE = re.compile(_LABEL + r'\s*:?\s*', re.I) NONE_RE = re.compile( - r'(?:none|nothing|nil|no|zero|0|n/?a|-+|—|–)' + r'[(\[]?\s*(?:none|nothing|nil|no|zero|0|n/?a|-+|—|–)' r'(?:\s+(?:found|open|left|remaining|seen|here|known|so\s+far|at\s+all|this\s+round|blocking|blockers?|majors?|' - r'findings?|issues?|problems?))*\s*(?:$|[.,;:!?)(\[])', re.I) + r'findings?|issues?|problems?))*\s*(?:$|[.,;:!?)(\[]|[^\w\s\x00-\x7f]|-(?=\s|$))', re.I) +FINDING_LINE_RE = re.compile(r'findings?\s*:\s*[\[(]?' + _KW + r'(?![^\W_])', re.I) # `finding: major x` FINDINGS_NONE_RE = re.compile(r'(?:findings|issues|problems)\s*:\s*(?:none|nothing|no|0|n/?a)\s*\.?\s*$', re.I) NOT_SAFE_RE = re.compile(r'\bnot\s+safe\b', re.I) VERDICT_NOT_SAFE_RE = re.compile(r'verdict\s*[:—–-]?\s*not\s+safe\b', re.I) RULE_LINE_RE = re.compile(r'\s*(?:[-*_]\s*){3,}') # a thematic break: no content HEADING_RE = re.compile(r' {0,3}#{1,6}(?:\s|$)') TABLE_RE = re.compile(r'\s*\|') -TABLE_RULE_RE = re.compile(r'\s*\|?\s*:?-{1,}:?\s*(?:\|\s*:?-{1,}:?\s*)*\|?\s*') +TABLE_RULE_RE = re.compile(r'\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?') # on a stripped line: linear time ROUND_RE = re.compile(r'r(\d{1,9})') SHOW_FINDING = 120 # characters of a finding line named in the reason BLOCKING = ('blocker', 'major') @@ -223,7 +231,7 @@ def judge(review, changes, config_path): def fold(text): """The text as the findings rules read it: NFKC, format characters (category Cf) gone, look-alike letters Latin.""" - text = unicodedata.normalize('NFKC', text or '') + text = re.sub(r'\\([!-/:-@\[-`{-~])', r'\1', html.unescape(unicodedata.normalize('NFKC', text or ''))) return ''.join(ch for ch in text if unicodedata.category(ch) != 'Cf').translate(CONFUSABLES) @@ -233,8 +241,9 @@ def _kind(kw): def _plain(line, narrow): - """One line normalised (lens.norm_line), folded, with leading symbols (emoji, arrows, bullets) gone.""" - return re.sub(r'^[^\w\[(]+', '', lens.norm_line(fold(line), narrow)).strip() + """One line normalised (lens.norm_line), folded, with leading symbols (emoji, arrows, bullets) gone and each run of + whitespace one space (the label patterns then take linear time on a long run of spaces).""" + return re.sub(r'\s+', ' ', re.sub(r'^[^\w\[(]+', '', lens.norm_line(fold(line), narrow))).strip() def _says_nothing(text): @@ -246,9 +255,9 @@ def _cells(raw): def _read(body, narrow): - """(findings [(line index, kind, text)], whether a blocker or major label was seen at all), one way of reading.""" + """(findings [(line index, kind, text)], the set of label kinds seen), one way of reading.""" lines = (body or '').splitlines() - out, seen, fence, columns = [], False, None, None + out, seen, fence, columns = [], set(), None, None for i, raw in enumerate(lines): fence, fenced = lens.fence_step(fence, raw) if fenced: @@ -256,16 +265,16 @@ def _read(body, narrow): continue if TABLE_RE.match(raw): cells = [_plain(c, narrow) for c in _cells(raw)] - if i + 1 < len(lines) and TABLE_RULE_RE.fullmatch(lines[i + 1]): # a header row: its label columns + if i + 1 < len(lines) and TABLE_RULE_RE.fullmatch(lines[i + 1].strip()): # a header row: its label columns columns = {j: _kind(m.group('kw')) for j, c in enumerate(cells) for m in [CELL_RE.fullmatch(c)] if m} - seen |= any(k in BLOCKING for k in columns.values()) + seen |= set(columns.values()) continue - if TABLE_RULE_RE.fullmatch(raw): + if TABLE_RULE_RE.fullmatch(raw.strip()): continue for j, c in enumerate(cells): # a label cell: the next cell says what it found m = CELL_RE.fullmatch(c) if m and _kind(m.group('kw')) in BLOCKING: - seen = True + seen.add(_kind(m.group('kw'))) said = next((x for x in cells[j + 1:] if x), '') if not _says_nothing(said): out.append((i, _kind(m.group('kw')), _plain(raw, narrow))) @@ -277,13 +286,19 @@ def _read(body, narrow): columns = None line = _plain(raw, narrow) if FINDINGS_NONE_RE.match(line): - seen = True + seen |= set(BLOCKING) + continue + m = FINDING_LINE_RE.match(line) + if m: # `finding: major x`: that format's section, and a finding of its kind + seen |= set(BLOCKING) + if _kind(m.group('kw')) in BLOCKING: + out.append((i, _kind(m.group('kw')), line)) continue if BARE_RE.fullmatch(line): # `### Major`: the first line under it says what it found kind = _kind(BARE_RE.fullmatch(line).group('kw')) if kind not in BLOCKING: continue - seen = True + seen.add(kind) under = _under(lines, i + 1, narrow) if under is None or not _says_nothing(under): out.append((i, kind, f'{line} (nothing under it)' if under is None else f'{line} {under}')) @@ -293,7 +308,7 @@ def _read(body, narrow): kind = _kind(m.group('kw') or m.group('kw2') or m.group('kw3')) if kind not in BLOCKING: continue - seen = True + seen.add(kind) said = line[m.end():marks[n + 1].start() if n + 1 < len(marks) else len(line)] if not _says_nothing(said): out.append((i, kind, line)) @@ -319,11 +334,12 @@ def _under(lines, start, narrow): def read_findings(body): """(findings, has_section). findings: [(kind, line)] of the report's blocker and major findings in line order, kind 'blocker' or 'major', line as normalised (cut to SHOW_FINDING). Read both ways lens reads a line (clean_inline - narrow or not): a line either way reads as a finding is one. has_section: the report has a blocker or major label - at all (`major: none` too), or `findings: none`. A first line (the lens header, or the title line under it) or a - `verdict:` line that says NOT SAFE is a blocker whatever the labels say.""" + narrow or not): a line either way reads as a finding is one. has_section: the report has a blocker label and a major + label (`blocker: none` and `major: none` too), or `findings: none`, or `finding: ` lines. A first line (the + lens header, or the title line under it) or a `verdict:` line that says NOT SAFE is a blocker whatever the labels + say.""" found = {} - seen = False + seen = set() for narrow in (False, True): out, saw = _read(body, narrow) seen |= saw @@ -340,7 +356,7 @@ def read_findings(body): line = _plain(raw, False) if (i in first and NOT_SAFE_RE.search(line)) or VERDICT_NOT_SAFE_RE.match(line): found[i] = ('blocker', f'the report says NOT SAFE: {line}'[:SHOW_FINDING]) - return [found[i] for i in sorted(found)], seen + return [found[i] for i in sorted(found)], seen >= set(BLOCKING) def findings(body): diff --git a/fork-ci/actions/pr-queue/review_judge.test.py b/fork-ci/actions/pr-queue/review_judge.test.py index 2f9737d9f15b6..3ac1b9212d6ba 100644 --- a/fork-ci/actions/pr-queue/review_judge.test.py +++ b/fork-ci/actions/pr-queue/review_judge.test.py @@ -4,7 +4,7 @@ are the contract both sides test; the rest here: reading git's name-status output, and that a rule set the parser would refuse, or a diff line nobody knows, never judges. """ -import dataclasses, json, os, sys, unittest +import dataclasses, json, os, sys, time, unittest sys.dont_write_bytecode = True # no __pycache__ next to the scripts @@ -168,6 +168,7 @@ def test_the_parser_refuses_those_rule_sets_first(self): BASIC = CONFIGS['basic'] # findings true, rounds approach_at 3, max 4, approach lens `approach`, no override_authors PEOPLE = dataclasses.replace(BASIC, override_authors=('a-person',)) HEAD = 'lens: code · reviewer: x · read at: ' + H1 + '\n' +NONE = 'blocker: none\nmajor: none' # the clean findings section the lens prompts ask for def rounds(history, reports, decided=False, review=BASIC, change_of=None): @@ -226,6 +227,16 @@ def rounds(history, reports, decided=False, review=BASIC, change_of=None): ('narrow read only', 'major (see '), # the wide tag eats the colon ('NOT SAFE header', '## Review lens A, head 0123abc: **NOT SAFE**\nblocker: none\nmajor: none'), ('NOT SAFE verdict', 'verdict: NOT SAFE\nblocker: none\nmajor: none'), + ('finding line', 'finding: major src/a.py:3 loses data'), # another project's lens format + ('finding line, blocker', '- finding: blocker the lock is gone'), + ('finding line after blocker: none', 'blocker: none\nfinding: major src/a.py:3 loses data'), + ('finding line after findings: none', 'findings: none\nfinding: major x'), + ('recap of a fixed major', NONE + '\n- Major 1 (cap): fixed at the root.'), # labelled: read as open + ('recap of a fixed blocker', NONE + '\n**Blocking 1:** fixed.'), + ('HTML entity', NONE + '\nmajor: x'), + ('Markdown escape', NONE + '\nmajor\\: x'), + ('ratio sign for a colon', NONE + '\nmajor\u2236 x'), + ('a dash glued to none', 'major: none-critical race'), ] # Lines a clean report writes: no finding, and a findings section. CLEAN = [ @@ -238,6 +249,7 @@ def rounds(history, reports, decided=False, review=BASIC, change_of=None): ('bold inline', '**Blocking:** none.\n**Major:** none.'), ('a rule between', '### Major\n\n---\n\nNone.'), ('table columns', '| blocker | major |\n|---|---|\n| none | none |'), + ('table columns, blanks round the rule', '| blocker | major |\n |---|---| \n| none | none |'), ('table row', '| severity | finding |\n|---|---|\n| major | none |'), ('n/a', 'major: n/a'), ('zero', 'major: 0'), ('em dash', 'major: —'), ('list', '- majors: none'), ('bold no', '**blockers:** no'), ('nothing', 'blocker: nothing'), ('NA', 'major: NA'), @@ -251,7 +263,24 @@ def rounds(history, reports, decided=False, review=BASIC, change_of=None): ('nothing found', 'major: nothing found'), ('a minor after it', 'major: none; minor: x'), ('prose that says not safe', 'major: none\nThe old code was not safe to run twice; this fixes it.'), + ('none, a dash, a note', 'major: none — two minors below'), + ('none, a hyphen, a note', 'major: none - two minors below'), + ('none in parentheses', 'Major: (none)'), + ('none and an emoji', 'major: None \u2705'), + ('only minor finding lines', 'finding: minor src/a.py:3 a typo\nfinding: nit y'), + ('a recap without a label', NONE + '\nround 1, fixed: the race'), ] +# Each makes a findings section on its own: a blocker and a major label, `findings: none` or `finding:` lines. +SECTION = ['Blocker: none, major: none', '### Blocking\nNone.\n\n### Major\nNone.', + '**Blocking:** none.\n**Major:** none.', '| blocker | major |\n|---|---|\n| none | none |', 'findings: none', + 'Blocker: none. Major: none.', + 'finding: minor a typo', NONE, '| severity | finding |\n|---|---|\n| blocker | none |\n| major | none |'] +# Review A and B of #740 round 2: `blocker: none` kept from the prompt, the major written in a form not read as a label. +# With one label kind only there is no findings section, so each fails closed (no section), never clean. +ONE_KIND = ['F1 (major): x', '1. **Retry drops rows** (major): x', '- F1. Severity: major. x', + '- F1 — **Severity:** Major — x', '- F1 — major: x', 'F1: major: x', 'One major: x', 'Major 1. x', + 'minor: typo. Also major: x'] +ONE_LABEL = ['major: none', 'blocker: none', '### Major\nNone.', '| severity | finding |\n|---|---|\n| major | none |'] # Lines that only mention a label: no finding, and no findings section either. MENTION = ['fixed: the major of round 1', 'no major found', 'majority: x', 'Major version: 2', 'blockerx: y', 'major 12345: too long a number', 'the blocking call in x'] @@ -265,10 +294,40 @@ def test_each_real_finding_format_counts(self): self.assertTrue(said, body) self.assertTrue(said[0][0] in ('blocker', 'major')) - def test_each_clean_format_is_no_finding_and_a_section(self): + def test_each_clean_format_is_no_finding(self): for what, body in CLEAN: with self.subTest(what): - self.assertEqual(rj.read_findings(HEAD + body), ([], True), body) + self.assertEqual(rj.read_findings(HEAD + body)[0], [], body) + self.assertEqual(rj.read_findings(HEAD + NONE + '\n\n' + body), ([], True), body) + + def test_a_section_needs_a_blocker_and_a_major_label(self): # #740 round 2, A major 1 + for body in SECTION: + with self.subTest(body): + self.assertEqual(rj.read_findings(HEAD + body), ([], True)) + for body in ONE_LABEL: + with self.subTest(body): + self.assertEqual(rj.read_findings(HEAD + body), ([], False)) + for body in ONE_KIND: + with self.subTest(body): + self.assertEqual(rj.read_findings(HEAD + 'blocker: none\n' + body), ([], False)) + r = rounds([('code', H1)], {'code': HEAD + 'blocker: none\n' + body}) + self.assertEqual(len(r.problems), 1) + self.assertIn('lens code: its newest report has no findings section', r.problems[0]) + + def test_a_long_run_of_spaces_takes_linear_time(self): # #740 round 2: 81-91 s for one 65,000-character line + n = 200_000 + for line in ('major' + ' ' * n + 'x', 'blocker: none' + ' ' * n + 'x', '| a | b |\n' + ' ' * n + 'x', + '| a | b |\n|' + ' ' * n + 'x', '| a | b |\n|-' + ' ' * n + 'x', + 'not' + ' ' * n + 'safe', 'finding:' + ' ' * n + 'x', 'major: none' + ' \t' * (n // 2) + 'x'): + with self.subTest(line[:12]): + start = time.monotonic() + rj.read_findings(HEAD + line) + self.assertLess(time.monotonic() - start, 2) + + def test_a_table_does_not_go_on_past_a_fence(self): # a fence ends the table, as GitHub renders it + table = '| blocker | major |\n|---|---|\n| none | none |\n' + self.assertEqual(rj.read_findings(table + '```\ncode\n```\n| a | b |'), ([], True)) + self.assertEqual(len(rj.findings(table + '| a | b |')), 1) # without the fence: a row of the table def test_a_line_that_only_mentions_a_label_is_neither(self): for line in MENTION: @@ -282,7 +341,7 @@ def test_the_kind_and_line_are_named(self): self.assertEqual(rj.findings('### Major')[0], ('major', 'Major (nothing under it)')) def test_an_unclosed_fence_runs_to_the_end(self): # GitHub shows the rest as code - self.assertEqual(rj.read_findings('major: none\n```\nmajor: x'), ([], True)) + self.assertEqual(rj.read_findings(NONE + '\n```\nmajor: x'), ([], True)) def test_fold(self): self.assertEqual(rj.fold('\uff2da\u200bj\u043er'), 'Major') @@ -323,7 +382,7 @@ def test_a_major_blocks_until_a_later_round_of_that_lens_clears_it(self): self.assertEqual(r.findings, (('code', 'major', 'major: two writers race'),)) self.assertIn('lens code reported a major that no later round cleared', r.problems[0]) self.assertIn('lens code posts a new report on the fixed head', r.problems[0]) - r = rounds([('code', H1), ('code r2', H2)], {'code': 'major: none'}) # round 2 cleared it + r = rounds([('code', H1), ('code r2', H2)], {'code': NONE}) # round 2 cleared it self.assertEqual((r.round, r.problems), (2, ())) def test_a_report_with_no_findings_section_fails_that_lens(self): # #99: never passes as clean @@ -332,7 +391,8 @@ def test_a_report_with_no_findings_section_fails_that_lens(self): # #99: never self.assertIn('lens code: its newest report has no findings section', r.problems[0]) self.assertIn('findings in inline review comments are not read', r.problems[0]) self.assertEqual(rounds([('code', H1)], {'code': HEAD}).problems, r.problems) # empty body: the same - self.assertEqual(rounds([('code', H1)], {'code': HEAD + 'major: none'}).problems, ()) + self.assertEqual(rounds([('code', H1)], {'code': HEAD + NONE}).problems, ()) + self.assertEqual(rounds([('code', H1)], {'code': HEAD + 'major: none'}).problems, r.problems) # one kind def test_an_entry_that_reads_nothing_under_an_agents_override_cannot_be_judged(self): r = rounds([('code', H1)], {'code': (rj.UNREAD, HEAD + 'major: none')}) @@ -353,16 +413,16 @@ def test_findings_false_judges_no_finding(self): self.assertIn('findings are not judged', r.text(dataclasses.replace(BASIC, findings=False))) def test_the_approach_lens_is_required_from_round_approach_at(self): - self.assertFalse(rounds([('code', H1), ('code', H2)], {'code': 'major: none'}).approach) - r = rounds([('code', H1), ('code', H2), ('code', H3)], {'code': 'major: none'}) + self.assertFalse(rounds([('code', H1), ('code', H2)], {'code': NONE}).approach) + r = rounds([('code', H1), ('code', H2), ('code', H3)], {'code': NONE}) self.assertTrue(r.approach) self.assertIn('approach lens approach required from round 3', r.text(BASIC)) self.assertFalse(rounds([('code', H1), ('code', H2), ('code', H3)], {}, decided=True).approach) def test_a_round_past_the_cap_stops_and_says_what_a_person_does(self): # #100 four = [('code', h) for h in (H1, H2, H3, H4)] - self.assertEqual(rounds(four, {'code': 'major: none'}).problems, ()) # round 4 of 4 may pass - r = rounds(four + [('code', H5)], {'code': 'major: none'}) + self.assertEqual(rounds(four, {'code': NONE}).problems, ()) # round 4 of 4 may pass + r = rounds(four + [('code', H5)], {'code': NONE}) self.assertTrue(r.over_cap) self.assertEqual(r.problems, ( 'review round 5 is past the cap of 4 rounds (issue #100): no more rounds; a person decides: close the pull ' @@ -370,10 +430,10 @@ def test_a_round_past_the_cap_stops_and_says_what_a_person_does(self): # #100 'override is accepted (issue #78: until agents no longer post as the person\'s login, an override is not ' 'proof of a person). A person\'s override alone is enough, also for a required approach lens that never ' 'posted',)) - r = rounds(four + [('code', H5)], {'code': 'major: none'}, review=PEOPLE) + r = rounds(four + [('code', H5)], {'code': NONE}, review=PEOPLE) self.assertIn('an override comment `override for head : ` for this head by a-person, newer ' 'than every lens comment', r.problems[0]) - self.assertEqual(rounds([('code r5', H1)], {'code': 'major: none'}).round, 5) + self.assertEqual(rounds([('code r5', H1)], {'code': NONE}).round, 5) def test_a_persons_decision_passes_the_cap_and_waives_the_approach_lens(self): five = [('code', h) for h in (H1, H2, H3, H4, H5)] @@ -399,10 +459,12 @@ def test_left_out_or_empty_is_no_person(self): # #78: agents still post as the def test_a_wrong_list_is_refused(self): block = VECTORS['configs']['basic'] - for bad in ('a-person', [''], ['-x'], ['a b'], ['x', 'X'], [1], ['x'] * 21, None): + for bad in ('a-person', [''], ['-x'], ['a b'], ['x', 'X'], [1], ['x'] * 21, [f'p{i}' for i in range(21)], None): with self.subTest(bad), self.assertRaises(config.ConfigError) as e: rc.parse_review({**block, 'override_authors': bad}) self.assertIn('override_authors', str(e.exception)) + self.assertEqual(len(rc.parse_review({**block, 'override_authors': [f'p{i}' for i in range(20)]}) + .override_authors), 20) # --- the fork's own config (issue #730): in shadow, workflow, queue and lens files critical (#82) ----------------