diff --git a/.github/fork-pr-queue.json b/.github/fork-pr-queue.json index a77cb870e66f9..b6c06c9f87d12 100644 --- a/.github/fork-pr-queue.json +++ b/.github/fork-pr-queue.json @@ -15,12 +15,32 @@ "head_check": "head_check", "review": { "version": 1, - "mode": "shadow", + "mode": "enforce", "rules": [ { "name": "gate", "weight": "critical", - "paths": [".github/**", "fork-ci/**"], + "paths": [ + ".github/*/*/**", + ".github/*.md", + ".github/*.td", + ".github/*.yml", + ".github/*.yaml", + ".github/fork-actions-policies.json", + ".github/fork-ci-release*.json", + ".github/fork-ci-rerun.json", + ".github/fork-fleet.json", + ".github/fork-owned-paths.txt", + ".github/fork-pr-queue.json", + ".github/fork-removed-workflows.txt", + ".github/fork-secret-sweep-gone-tips.txt", + ".github/fork-tests-upstream.json", + ".github/fork-upstream.json", + "docs/fork/gate-trust.md", + "fork-ci/**", + "fork-kit/gh-apps/**", + "fork-kit/review/**" + ], "lenses": ["merge-path", "state"] }, { "name": "code", "weight": "normal", "paths": ["**"], "lenses": ["code"] } @@ -34,6 +54,7 @@ "min_lenses": { "critical": 2, "normal": 1, "light": 0 }, "rounds": { "approach_at": 3, "max": 4, "approach_lens": "approach" }, "findings": true, + "override_authors": [], "state_line": false } } diff --git a/apps/server/src/fork/reviewRules/plan.ts b/apps/server/src/fork/reviewRules/plan.ts index 5b3942c5b7863..24287b7d339cf 100644 --- a/apps/server/src/fork/reviewRules/plan.ts +++ b/apps/server/src/fork/reviewRules/plan.ts @@ -109,8 +109,8 @@ export interface ReviewPlan { /** * The plan for `changes` as `git diff --name-status -M` names them ([status, path] or [R|C, old, new]) under a * valid `review` block; `configPath` is the config file (repo-relative). A rename or copy counts under both paths; a - * protected path also counts for the first critical rule; the heaviest weight hit decides; no change at all counts as - * the catch-all rule. + * protected path also counts for the first critical rule; the heaviest weight hit is the weight; the lenses are those + * of every rule hit, in rule order, each once; no change at all counts as the catch-all rule. */ export function planReview( review: ReviewBlock, @@ -165,8 +165,9 @@ export function planReview( : []; }); const weight = WEIGHTS.find((w) => hits.some((h) => h.weight === w))!; + // Every rule hit gives its lenses, whatever its weight (owner decision: fork issue 82, one extra review). const lenses = review.rules - .filter((r) => r.weight === weight && hits.some((h) => h.rule === r.name)) + .filter((r) => hits.some((h) => h.rule === r.name)) .flatMap((r) => r.lenses); return { weight, lenses: [...new Set(lenses)], hits, protected: isProtected }; } diff --git a/fork-ci/RELEASE-NOTES.md b/fork-ci/RELEASE-NOTES.md index f3ec50d85c707..d4dfa92fcc1f1 100644 --- a/fork-ci/RELEASE-NOTES.md +++ b/fork-ci/RELEASE-NOTES.md @@ -13,6 +13,20 @@ nothing. ## After ci/v14 +### pr-queue and head-check: every rule hit gives its lenses (#82) + +With a `review` block in `enforce`, the required lenses are now the lenses of every rule a changed path counted for, +whatever its weight, not only those of the rules of the heaviest weight hit. An app change plus one critical file needs +the app rule's lens and the critical lenses; before, the critical lenses alone passed it and no lens read the app code. +The weight is still the heaviest hit. T3's copy of the rules (`apps/server/src/fork/reviewRules`) plans the same lenses. +First in the release after `ci/v22`. + +**Upgrade:** a config in `enforce` needs more lens reads on mixed changes after the move, and a built-in protected path +(the config, a lens prompt, a workflow) under a lighter rule also needs that rule's lenses. To keep the protected paths +at the critical lenses only, put them in a critical rule. In `shadow` only the summary row changes. `ci/v21` and `ci/v22` +accept the same config but judge mixed changes the old, looser way, so a caller of a config in `enforce` pins this release +or a later one (pr-queue README, "Turning it on: release, pin, then config", step 1). + ### pr-queue and head-check: review findings, rounds and `override_authors` (#740) With a `review` block, the gate also judges the review itself from the lens comments. With `findings: true`, a lens's diff --git a/fork-ci/actions/head-check/README.md b/fork-ci/actions/head-check/README.md index 2ecab8ea19852..702c74de21c3f 100644 --- a/fork-ci/actions/head-check/README.md +++ b/fork-ci/actions/head-check/README.md @@ -45,9 +45,10 @@ All of them are the queue's, imported from `../pr-queue`, never copied: - **Lenses by changed path** (`review_judge.py`, the pr-queue README, "Review rules"): when the base's config has a `review` block, the paths the head changes against its merge base (`git diff --name-status -M`, renames under - both paths, no cap) are judged by its rules. In `enforce` the lenses of the - heaviest weight hit are required on top of `required-lenses`; in `shadow` the - summary only adds a row "the path rules would have said: ...". A change the + both paths, no cap) are judged by its rules. In `enforce` the lenses of every + rule hit (whatever its weight) are required on top of `required-lenses`; in + `shadow` the summary only adds a row "the path rules would have said: ...". + A change the rules cannot judge (no merge base, a failed diff) refuses, in either mode. Without the block, nothing of this runs. - **Findings and rounds** (`review_judge.judge_rounds`, the pr-queue README, diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index a55c12804d0cc..6902dee39ee6f 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -695,15 +695,18 @@ def test_shadow_only_says_what_the_rules_would_have_needed(self): self.assertEqual(code, 1, out) self.assertIn('required lens without a read: design', out) - def test_enforce_needs_the_lenses_of_the_heaviest_rule_hit(self): + def test_enforce_needs_the_lenses_of_every_rule_hit(self): # #82: the critical lenses never replace code's base = self.base() head = self.head(base, **{'src/storage/s.py': lines(40, ['x'], 3), 'src/app.py': 'a\n', 'docs/d.md': 'd\n'}) code, out = self.check(base, head, ['code']) self.assertEqual(code, 1, out) self.assertIn('required lens without a read: data-1, data-2', out) - self.assertIn('path rules (enforce): the path rules decide: critical, lenses data-1, data-2 (3 changed paths', - out) - self.assertEqual(self.check(base, head, ['data-1', 'data-2'])[0], 0) + self.assertIn('path rules (enforce): the path rules decide: critical, lenses data-1, data-2, code (3 changed ' + 'paths', out) + code, out = self.check(base, head, ['data-1', 'data-2']) + self.assertEqual(code, 1, out) + self.assertIn('required lens without a read: code', out) + self.assertEqual(self.check(base, head, ['data-1', 'data-2', 'code'])[0], 0) def test_enforce_still_requires_the_input_lenses(self): base = self.base() @@ -748,24 +751,28 @@ def test_a_rename_counts_under_its_old_and_its_new_path(self): self.assertEqual(code, 1, out) self.assertIn('data-1, data-2', out) head = self.head(base, drop=['src/storage/s.py'], **{'src/s.py': lines(40)}) - code, out = self.check(base, head, ['data-1', 'data-2']) + code, out = self.check(base, head, ['data-1', 'data-2', 'code']) # the new path is code's to read (#82) self.assertEqual(code, 0, out) self.assertIn('data (critical): src/storage/s.py; code (normal): src/s.py', out) def test_the_config_and_the_lens_prompts_are_always_critical(self): base = self.base() prompt = REVIEW_BLOCK['lenses']['code']['prompt'] - for name, kw in (('config edited', {'cfg': review_cfg('enforce', findings=False)}), # deleted: refused (#388) - ('prompt renamed', {'drop': [prompt], 'docs/code-lens.md': PROMPTS[prompt]}), - ('prompt edited', {prompt: 'changed\n'}), - ('workflow', {'.github/workflows/ci.yml': 'on: push\n'})): - with self.subTest(name): + # A protected path also counts for its own rule: under the catch-all (the config, a workflow) it needs the + # code lens too, every rule hit gives its lenses (#82); a prompt under the light docs rule adds none. + for name, kw, own in (('config edited', {'cfg': review_cfg('enforce', findings=False)}, ['code']), + ('prompt renamed', {'drop': [prompt], 'docs/code-lens.md': PROMPTS[prompt]}, []), + ('prompt edited', {prompt: 'changed\n'}, []), + ('workflow', {'.github/workflows/ci.yml': 'on: push\n'}, ['code'])): + with self.subTest(name): # the config deleted: refused (#388) head = self.head(base, **kw) code, out = self.check(base, head, ['code']) self.assertEqual(code, 1, out) - self.assertIn('critical, lenses data-1, data-2', out) + self.assertIn('critical, lenses ' + ', '.join(['data-1', 'data-2', *own]) + ' (', out) self.assertIn('protected: ', out) - self.assertEqual(self.check(base, head, ['data-1', 'data-2'])[0], 0) + self.assertEqual(self.check(base, head, ['data-1', 'data-2', *own])[0], 0) + if own: + self.assertEqual(self.check(base, head, ['data-1', 'data-2'])[0], 1) def test_an_empty_diff_is_judged_by_the_catch_all_rule(self): # empty state: never "no lens needed" base = self.base() @@ -1538,6 +1545,160 @@ class FindingsAndRoundsBare(FindingsAndRounds): BARE = True +# --- the fork's own config: .github/fork-pr-queue.json in enforce (#82, #99, #100; no override author, #78) --------- +FORK_ROOT = os.path.join(HERE, '..', '..', '..') +FORK_CONFIG_PATH = '.github/fork-pr-queue.json' +with open(os.path.join(FORK_ROOT, FORK_CONFIG_PATH), encoding='utf-8') as _f: + FORK_RAW = json.load(_f) + + +def fork_prompts(): + out = {} + for x in FORK_RAW['review']['lenses'].values(): + with open(os.path.join(FORK_ROOT, x['prompt']), encoding='utf-8') as f: + out[x['prompt']] = f.read() + return out + + +class ForkConfigRules(unittest.TestCase): + """The gate on a base that holds the fork's real config and lens prompts: each rule the owner decided blocks when it + should and passes when it should, and the same config in shadow only reports. A test that fails here after a + config change means the change alters what the fork's gate decides.""" + BARE = False + FILES = {'apps/server/src/x.ts': lines(), '.github/workflows/ci.yml': lines(), 'fork-ci/actions/x.py': lines(), + '.github/fork-seams.json': lines(), '.github/fork-ci-tests.txt': lines()} + + def setUp(self): + self.w = World(self.BARE) + self.addCleanup(self.w.close) + self.p = Posts() + + def pair(self, *changed, mode=None): + """A base with the fork's config (its mode replaced by `mode`) and a head on it that changes `changed`.""" + raw = json.loads(json.dumps(FORK_RAW)) + if mode: + raw['review']['mode'] = mode + tree = {FORK_CONFIG_PATH: json.dumps(raw, indent=2) + '\n', **fork_prompts(), **self.FILES} + base = self.w.commit(tree) + return base, self.w.commit({**tree, **{p: lines(30, ['x'], 3) for p in changed or ('apps/server/src/x.ts',)}}, + [base]) + + def check(self, base, head, comments): + path = self.w.checkout(base, head) + return run_cli(path, fake_gh((0, json.dumps(pr(comments, head=head)))), config_path=FORK_CONFIG_PATH) + + def said(self, name, sha, *lines): + return FindingsAndRounds.said(self, name, sha, *lines) + + def override(self, sha): # from the owner's login, which agents post from too (#78) + return self.p.post(f"{FORK_RAW['override']['prefix']} {sha}: read it myself") + + def test_it_is_in_enforce(self): + self.assertEqual(FORK_RAW['review']['mode'], 'enforce') + # #78: no person posts from a login of their own yet. Written out, empty: releases ci/v20 and older do not know + # the key and refuse the config (fail closed), where they would judge no finding and no round (fail open). + self.assertEqual(FORK_RAW['review']['override_authors'], []) + + def test_a_workflow_or_gate_change_needs_the_extra_review(self): # #82 + for changed in ('.github/workflows/ci.yml', 'fork-ci/actions/x.py', 'docs/fork/gate-trust.md'): + with self.subTest(changed): + base, head = self.pair(changed) + code, out = self.check(base, head, [self.said('code', head)]) + self.assertEqual(code, 1, out) + self.assertIn('required lens without a read: merge-path, state', out) + code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)]) + self.assertEqual(code, 0, out) + code, out = self.check(base, head, [self.said('merge-path', head)]) # one review is not enough + self.assertEqual(code, 1, out) + + def test_an_app_change_plus_a_gate_file_needs_the_code_lens_too(self): # #82: one EXTRA review, not instead + for gate in ('.github/workflows/ci.yml', 'fork-ci/actions/x.py'): + with self.subTest(gate): + base, head = self.pair('apps/server/src/x.ts', gate) + code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)]) + self.assertEqual(code, 1, out) # nobody read the app code + self.assertIn('required lens without a read: code', out) + self.assertIn('the path rules decide: critical, lenses merge-path, state, code (', out) + code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head), + self.said('code', head)]) + self.assertEqual(code, 0, out) + + def test_the_rollback_to_shadow_needs_the_gate_lenses(self): # it changes the config: judged by the base's rules + raw = json.loads(json.dumps(FORK_RAW)) + raw['review']['mode'] = 'shadow' + base, _ = self.pair() + tree = {FORK_CONFIG_PATH: json.dumps(FORK_RAW, indent=2) + '\n', **fork_prompts(), **self.FILES} + head = self.w.commit({**tree, FORK_CONFIG_PATH: json.dumps(raw, indent=2) + '\n'}, [base]) + code, out = self.check(base, head, [self.said('code', head)]) + self.assertEqual(code, 1, out) + self.assertIn('required lens without a read: merge-path, state', out) + code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)]) + self.assertEqual(code, 0, out) + + def test_an_app_change_that_adds_a_seam_or_a_test_line_needs_the_code_lens(self): # the dry run's wrong refusals + for extra in ('.github/fork-seams.json', '.github/fork-ci-tests.txt'): + with self.subTest(extra): + base, head = self.pair('apps/server/src/x.ts', extra) + code, out = self.check(base, head, [self.said('code', head)]) + self.assertEqual(code, 0, out) + self.assertIn('path rules (enforce): the path rules decide: normal, lenses code (', out) + code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)]) + self.assertEqual(code, 1, out) # the gate lenses alone never read app code + self.assertIn('required lens without a read: code', out) + + def test_a_blocker_or_major_fails_and_the_owners_login_does_not_clear_it(self): # #99, #78 + base, head = self.pair() + self.assertEqual(self.check(base, head, [self.said('code', head)])[0], 0) + for line in ('major: two writers race', 'blocker: data lost on restart'): + with self.subTest(line): + code, out = self.check(base, head, [self.said('code', head, line)]) + self.assertEqual(code, 1, out) + self.assertIn(f"that no later round cleared (issue #99): '{line}'", out) + code, out = self.check(base, head, [self.said('code', head, line), self.override(head)]) + self.assertEqual(code, 1, out) + code, out = self.check(base, head, [self.said('code', head, 'Looks good to me.')]) # no findings section + self.assertEqual(code, 1, out) + self.assertIn('has no findings section', out) + + def test_round_three_needs_the_approach_lens_and_round_five_is_past_the_cap(self): # #100 + base, head = self.pair() + two = [self.said('code', OLD[0]), self.said('code r2', head)] + self.assertEqual(self.check(base, head, two)[0], 0) + three = [self.said('code', OLD[0]), self.said('code r2', OLD[1]), self.said('code r3', head)] + code, out = self.check(base, head, three) + self.assertEqual(code, 1, out) + self.assertIn('required lens without a read: approach', out) + code, out = self.check(base, head, three + [self.said('approach', head)]) + self.assertEqual(code, 0, out) + self.assertIn('round 3 of 4, approach lens approach required from round 3', out) + five = [self.said('code', sha) for sha in OLD] + [self.said('code', head), self.said('approach', head)] + code, out = self.check(base, head, five) + self.assertEqual(code, 1, out) + self.assertIn('review round 5 is past the cap of 4 rounds (issue #100)', out) + code, out = self.check(base, head, five + [self.override(head)]) # no override author: no person decides + self.assertEqual(code, 1, out) + self.assertIn('no one is listed in review.override_authors yet, so no override is accepted', out) + + def test_shadow_only_reports_what_enforce_refuses(self): # the rollback: the same config in shadow + base, head = self.pair('.github/workflows/ci.yml', mode='shadow') + code, out = self.check(base, head, [self.said('code', head)]) + self.assertEqual(code, 0, out) + self.assertIn('path rules (shadow): the path rules would have said: critical, lenses merge-path, state', out) + base, head = self.pair(mode='shadow') + code, out = self.check(base, head, [self.said('code', head, 'major: two writers race')]) + self.assertEqual(code, 0, out) + self.assertIn('findings and rounds (shadow): they would have said: round 1 of 4; lens code reported a major', + out) + five = [self.said('code', sha) for sha in OLD] + [self.said('code', head)] + code, out = self.check(base, head, five) + self.assertEqual(code, 0, out) + self.assertIn('findings and rounds (shadow): they would have said: round 5 of 4', out) + + +class ForkConfigRulesBare(ForkConfigRules): + BARE = True + + class FetchTimeout(unittest.TestCase): # #740 round 2, B minor 4: a hung fetch of a read commit never hangs the check def test_a_fetch_that_takes_too_long_failed(self): timeouts = [] diff --git a/fork-ci/actions/head-check/review_guide.test.py b/fork-ci/actions/head-check/review_guide.test.py index 8b193f12ef6ff..90e8f1a2f2f5a 100644 --- a/fork-ci/actions/head-check/review_guide.test.py +++ b/fork-ci/actions/head-check/review_guide.test.py @@ -192,10 +192,23 @@ def test_findings_rounds_and_override_authors_come_after_the_same_release(self): first = re.search(r'First in the release after `(ci/v\d+)`', entry) self.assertIsNotNone(first, entry) text = ' '.join(guide().split()) - self.assertIn(f'Findings, rounds and `override_authors` are in the first release after `{first[1]}`', text) + after = f'ci/v{int(first[1][len("ci/v"):]) + 1}' # the release after it (ReleaseTags checks the tags) + self.assertIn(f'Findings, rounds and `override_authors` are in `{after}` and later', text) self.assertIn(f'`{first[1]}` or older for `override_authors`', text) self.assertIn(f'`{first[1]}` and older refuse a config with `override_authors`', entry) + def test_a_caller_in_enforce_pins_the_release_with_every_rule_hit(self): # #82: the floor, in step with the notes + entry = self.entry(82) + first = re.search(r'First in the release after `(ci/v\d+)`', entry) + self.assertIsNotNone(first, entry) + older = f'ci/v{int(first[1][len("ci/v"):]) - 1}' + floor = f'pins the first release after `{first[1]}` or a later one' + self.assertIn(f'Every rule hit gives its lenses from the first release after `{first[1]}` (#82)', flat(step(1))) + self.assertIn(f'So a caller of a config in `enforce` {floor}: `{older}` and `{first[1]}` accept the config but ' + 'judge mixed changes the old, looser way', flat(step(1))) + self.assertIn(f'A caller of this config {floor} (step 1)', flat(step(5))) # the fork's own caller (#81) + self.assertIn(f'a caller of a config in `enforce` pins this release or a later one', entry) + def test_the_release_notes_upgrade_points_to_the_guide(self): self.assertIn(f'pr-queue README, "{HEADING}"', self.entry(740)) @@ -407,10 +420,14 @@ def test_a_pin_back_is_checked_by_nothing_but_review(self): def test_the_enforce_pull_request_needs_only_the_input_lenses(self): """Step 5: the pull request that sets enforce is judged under the base's shadow, so the gate requires only the - input's lenses, not the critical ones its own change (the config, a protected path) would need in enforce.""" - self.assertIn('so ask for the critical lenses: the gate does not require them yet', step(5)) + input's lenses, not the ones its own change would need in enforce: the critical lenses (the config is a + protected path) plus the lenses of the rule the config sits under (here the `code` catch-all, #82).""" + self.assertIn('so ask for the critical lenses plus the lenses of every other rule its paths hit (a config ' + "under a normal catch-all also needs that rule's lens): the gate does not require them yet", + flat(step(5))) judgement = review_judge.judge(parsed('shadow').review, [('M', (CONFIG_PATH,))], CONFIG_PATH) - self.assertEqual((judgement.weight, judgement.lenses), ('critical', ('data-1', 'data-2'))) + self.assertEqual((judgement.weight, judgement.lenses), ('critical', ('data-1', 'data-2', 'code'))) + self.assertEqual([h.rule for h in judgement.hits], ['data', 'code']) for mode, want in (('shadow', {'code'}), ('enforce', {'code', 'data-1', 'data-2'})): seen = [] run_decide(mode, rounds=(review_judge.Rounds(1, False, False, (), ()), None), judgement=judgement, diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index 73f1731530b4c..ea2b377c8e92a 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -141,8 +141,15 @@ whole). Then (`review_judge.py`, `review_glob.py`): folder is a critical change); a delete, a mode change and a type change count under their path; - a built-in protected path also counts for the first critical rule; -- the heaviest weight hit decides; the required lenses are the lenses of every - rule of that weight that was hit, in rule order, each once; +- the heaviest weight hit is the change's weight; the required lenses are the + lenses of every rule that was hit, whatever its weight, in rule order, each + once. Each path gets its own rule's read: an app change plus one workflow + file needs the app rule's lens and the critical lenses, never the critical + lenses alone (owner decision, + [issue 82](https://github.com/scuttletoby/t3code/issues/82): one _extra_ + review). A protected path counts for its own rule too, so one under a + normal catch-all also needs that rule's lens; put the protected paths in a + critical rule of their own to keep them at the critical lenses; - an empty diff (the head adds nothing to its merge base) counts as the catch-all rule, never as "no lens needed"; - with more than one merge base (criss-cross merges), the paths changed against @@ -311,8 +318,14 @@ pull request of its own on the default branch. ([fork-ci README, "How a release is made"](../../README.md#how-a-release-is-made)); a project never makes one. The path rules (`review`, `shadow` and `enforce`) are in `ci/v15` and later. Findings, rounds and `override_authors` are in - the first release after `ci/v20` - ([RELEASE-NOTES.md](../../RELEASE-NOTES.md), #740). Take the newest + `ci/v21` and later + ([RELEASE-NOTES.md](../../RELEASE-NOTES.md), #740). Every rule hit gives + its lenses from the first release after `ci/v22` (#82); up to `ci/v22` only + the lenses of the heaviest weight hit were required. So a caller of a + config in `enforce` pins the first release after `ci/v22` or a later one: + `ci/v21` and `ci/v22` accept the config but judge mixed changes the old, + looser way (an app change plus one critical file passes on the critical + lenses alone, and no lens reads the app code). Take the newest `ci/vN` and its release sha as in [Pinning](../../README.md#pinning). 2. **Pin.** In one pull request, move every `uses:` of fork-ci in the @@ -342,9 +355,7 @@ pull request of its own on the default branch. request. Keep the caller's `required-lenses` as it is: in `shadow` it still decides. This pull request is judged by the old rules (the gate reads the config from the base tip), and its merged config must parse, or - the gate refuses it. The fork's own config, - [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), is at - this step. + the gate refuses it. 4. **Read the results.** From now on every gate run adds rows to the summary of the gate App's check run on the head ("Details" on the pull request, or @@ -371,7 +382,9 @@ pull request of its own on the default branch. 5. **Enforce.** When the rows have matched the real verdicts over the number of merged pull requests the project picked, and no row shows a config warning, one pull request sets `"mode": "enforce"`. It changes the - config, so ask for the critical lenses: the gate does not require them yet + config, so ask for the critical lenses plus the lenses of every other + rule its paths hit (a config under a normal catch-all also needs that + rule's lens): the gate does not require them yet (the base is still in `shadow`, where only the input decides). Dispatch the backfill right after it merges, and arm no pull request before the backfill has run: each open pull request is judged by the new rules only at @@ -401,6 +414,45 @@ pull request of its own on the default branch. } ``` + The fork's own config, + [`.github/fork-pr-queue.json`](../../../.github/fork-pr-queue.json), is at + this step (owner decisions in issues + [82](https://github.com/scuttletoby/t3code/issues/82), + [99](https://github.com/scuttletoby/t3code/issues/99) and + [100](https://github.com/scuttletoby/t3code/issues/100)). Its + `override_authors` is written out empty: no person while agents post from + the owner's login + ([issue 78](https://github.com/scuttletoby/t3code/issues/78)), and + `ci/v20` and older refuse the key, so a caller pinned to one of them fails + closed instead of enforcing the path rules without findings and rounds. A + caller of this config pins the first release after `ci/v22` or a later + one (step 1): `ci/v21` and `ci/v22` accept it but judge mixed changes the + old, looser way. The fork's gate caller + ([issue 81](https://github.com/scuttletoby/t3code/issues/81)) starts at + that floor. Its critical + rule names the workflow and gate-setting files under `.github` one by one + instead of `.github/**`: the seam list and the test list, which most pull + requests add a line to, stay normal, or plain app changes would need the + gate lenses and no `code` lens. It also holds `fork-ci/**`, + `fork-kit/review/**` (the lens format checker and lens skill: what a lens + report must say) and `fork-kit/gh-apps/**` (the gate App's key and + tokens), and `docs/fork/gate-trust.md` (the owner's paste commands that + set up that key and those tokens); the rest of `docs/fork` stays normal. + Tests in `review_judge.test.py` fail when a new file under + `.github` or a new folder under `fork-kit` is not sorted. The fork runs no + gate on its own pull requests yet (no caller, App or ruleset, + [issue 81](https://github.com/scuttletoby/t3code/issues/81)), so the block + decides nothing there until one exists. + + Until issue 78 is done, two of these rules are not yet a person's + decision on the fork. The round cap: rounds count from the lens comments + that still exist, and every lens author is the owner's login that agents + post from, so an agent can delete old round comments (or open a new pull + request) and start again at round 1. A push after the review: an override + for the new head lets it pass on the old reads without a listed person + (open findings still block). Only clearing a finding and passing the cap + need a listed person, and none is listed. + 6. **Roll back** in the reverse order: lens names back into an emptied input, then `mode` back to `shadow` (or the block out), and only then a pin back. Never pin a release that does not know a key the config still @@ -410,6 +462,17 @@ pull request of its own on the default branch. key, so it can merge; after it, the old release refuses the config, every head fails, and only the owner step below gets out. + The fork's own rollback to `shadow` is one pull request with three + changes: the `"mode"` line of `.github/fork-pr-queue.json`, and the two + tests that pin `enforce`, `ForkConfigRules.test_it_is_in_enforce` in + `head_check.test.py` and + `ForkConfig.test_it_parses_in_enforce_with_the_owners_numbers_and_no_person_yet` + in `review_judge.test.py`. While the base is in `enforce`, that pull + request is judged by the enforce rules: it changes the config and + `fork-ci/**`, so it needs clean `merge-path` and `state` reports within + the round cap. If the gate itself is broken so that every head fails, only + the owner step below gets out. + **Stopped half-way, or run again.** Every step leaves a state that works or fails closed: diff --git a/fork-ci/actions/pr-queue/review.vectors.json b/fork-ci/actions/pr-queue/review.vectors.json index 236e275ced02c..6a065e72e39dc 100644 --- a/fork-ci/actions/pr-queue/review.vectors.json +++ b/fork-ci/actions/pr-queue/review.vectors.json @@ -255,7 +255,7 @@ "protected": [] }, { - "name": "one critical file among normal ones needs the critical lenses only", + "name": "one critical file among normal ones needs the critical lenses and the code lens", "config": "basic", "changes": [ ["M", "src/storage/db.py"], @@ -263,7 +263,7 @@ ["M", "src/app/b.py"] ], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "code"], "protected": [] }, @@ -272,7 +272,7 @@ "config": "basic", "changes": [["R100", "src/storage/old.py", "src/app/new.py"]], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "code"], "protected": [] }, @@ -281,7 +281,7 @@ "config": "basic", "changes": [["R087", "src/app/a.sql", "migrations/0007.sql"]], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "code"], "protected": [] }, @@ -299,7 +299,7 @@ "config": "basic", "changes": [["C075", "src/storage/a.py", "src/app/a.py"]], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "code"], "protected": [] }, @@ -320,7 +320,7 @@ "config": "basic", "changes": [["M", ".github/pr-queue.json"]], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "code"], "protected": [".github/pr-queue.json"] }, @@ -332,7 +332,7 @@ ["M", "docs/guide.md"] ], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "docs", "code"], "protected": [".github/pr-queue.json"] }, @@ -371,7 +371,7 @@ ["M", ".github/workflows/sub/inner.yml"] ], "weight": "critical", - "lenses": ["data-1", "data-2"], + "lenses": ["data-1", "data-2", "code"], "hits": ["data", "code"], "protected": [".github/workflows/deploy.yml", ".github/workflows/sub/inner.yml"] }, @@ -409,7 +409,7 @@ "protected": [] }, { - "name": "two critical rules hit: the lenses of both", + "name": "two critical rules and a normal one hit: the lenses of all three", "config": "two-critical", "changes": [ ["M", "src/auth/login.py"], @@ -417,7 +417,7 @@ ["M", "src/api/users.py"] ], "weight": "critical", - "lenses": ["security-1", "security-2", "data-1", "data-2"], + "lenses": ["security-1", "security-2", "data-1", "data-2", "api", "code"], "hits": ["auth", "data", "api"], "protected": [] }, @@ -486,7 +486,7 @@ ["M", "db/x.sql"] ], "weight": "critical", - "lenses": ["security-1", "security-2", "data-1", "data-2"], + "lenses": ["security-1", "security-2", "data-1", "data-2", "code"], "hits": ["auth", "data", "rest"], "protected": ["review/data.md"] }, @@ -498,7 +498,7 @@ ["M", ".github/pr-queue.json"] ], "weight": "critical", - "lenses": ["security-1", "security-2"], + "lenses": ["security-1", "security-2", "code"], "hits": ["auth", "notes", "rest"], "protected": [".github/pr-queue.json"] }, diff --git a/fork-ci/actions/pr-queue/review_judge.py b/fork-ci/actions/pr-queue/review_judge.py index 2fa0aee0576ec..3c4bdbfa0da27 100644 --- a/fork-ci/actions/pr-queue/review_judge.py +++ b/fork-ci/actions/pr-queue/review_judge.py @@ -9,8 +9,10 @@ - Built-in protected paths, not configurable: the config file itself, every lens prompt file named in `lenses`, and `.github/workflows/**`. Such a path also counts for the first critical rule of the list, whatever rule it belongs to. - - The heaviest weight hit decides. The required lenses are the lenses of every rule of that weight that was hit, in - the order of the rules and of each rule's list, each name once. + - The heaviest weight hit is the change's weight. The required lenses are the lenses of every rule that was hit, + whatever its weight, in the order of the rules and of each rule's list, each name once: each rule's paths get + their own rule's read, so an app change plus one gate file needs the code lens and the gate lenses (owner + decision: issue #82, "one extra review"), never the gate lenses alone. - No changed path at all (an empty diff) counts as the catch-all rule: the change is judged as a new folder would be, never as "no lens needed". Unknown never judges: a path no rule matches, a protected path with no critical rule (the parser refuses both, so @@ -169,8 +171,7 @@ def judge(review, changes, config_path): hits = tuple(Hit(r.name, r.weight, tuple(counted[r.name] or ())) for r in review.rules if counted[r.name] is None or counted[r.name]) weight = next(w for w in review_config.WEIGHTS if any(h.weight == w for h in hits)) - lenses = tuple(dict.fromkeys(n for r in review.rules if r.weight == weight - and any(h.rule == r.name for h in hits) for n in r.lenses)) + lenses = tuple(dict.fromkeys(n for r in review.rules if any(h.rule == r.name for h in hits) for n in r.lenses)) return Judgement(weight, lenses, hits, tuple(protected), len(paths)) diff --git a/fork-ci/actions/pr-queue/review_judge.test.py b/fork-ci/actions/pr-queue/review_judge.test.py index 021d1fa711854..7d11ebb684096 100644 --- a/fork-ci/actions/pr-queue/review_judge.test.py +++ b/fork-ci/actions/pr-queue/review_judge.test.py @@ -71,7 +71,7 @@ def test_the_same_input_gives_the_same_judgement(self): # runs again: nothing def test_the_summary_line_names_weight_lenses_and_why(self): case = next(c for c in VECTORS['cases'] if c['name'] == 'the config file itself is critical') text = judged(case).text() - self.assertTrue(text.startswith('critical, lenses data-1, data-2 (1 changed path; '), text) + self.assertTrue(text.startswith('critical, lenses data-1, data-2, code (1 changed path; '), text) self.assertIn('protected: .github/pr-queue.json', text) empty = next(c for c in VECTORS['cases'] if c['changes'] == [] and c['config'] == 'basic') self.assertIn('code (normal): no changed path', judged(empty).text()) @@ -427,9 +427,21 @@ def test_a_wrong_list_is_refused(self): .override_authors), 20) -# --- the fork's own config (issue #730): in shadow, workflow, queue and lens files critical (#82) ---------------- +# --- the fork's own config (issue #730): in enforce, workflow, queue, lens and other CI files critical (#82) --------- ROOT = os.path.join(HERE, '..', '..', '..') FORK_CONFIG = '.github/fork-pr-queue.json' +# The two .github files that are not workflow files or gate settings (#82): per-pull-request bookkeeping lists a CI +# check compares with the tree (fork-seam-check.cjs: every fork line in an upstream file is announced; fork-ci-tests.cjs: +# every test file is listed). Most fork pull requests add a line to one of them; as critical files they made the gate +# lenses (merge-path, state) the only required ones on plain app changes, and no lens read the code (the dry run of +# the switch, issue #82). Every other file under .github stays critical: a new one fails this test until it is sorted. +NOT_GATE = ('.github/fork-seams.json', '.github/fork-ci-tests.txt') +# fork-kit folders that are gate settings (#82): review is the lens-format checker and the lens skill (what a lens +# report must say to count), gh-apps keeps the gate App's key and mints its tokens (the gate-trust design). The +# other kits run machines and agents, and the merge gate reads none of them. +GATE_KITS = ('gh-apps', 'review') +OTHER_KITS = ('.gitignore', 'README.md', 'agent-guards', 'cleaner', 'common', 'desk', 'golive', 'heavy', 'model-guard', + 'pc-agents', 'schedule-kick') class ForkConfig(unittest.TestCase): @@ -437,29 +449,94 @@ def setUp(self): with open(os.path.join(ROOT, FORK_CONFIG), encoding='utf-8') as f: self.cfg = config.parse(f.read(), FORK_CONFIG) self.review = self.cfg.review + self.critical = next(x for x in self.review.rules if x.weight == 'critical') - def test_it_parses_in_shadow_with_the_owners_numbers_and_no_person_yet(self): - r = self.review # shadow first (README, Turning it on); no override author until #78 + def judge(self, path): + return rj.judge(self.review, [('M', (path,))], FORK_CONFIG) + + def test_it_parses_in_enforce_with_the_owners_numbers_and_no_person_yet(self): + r = self.review # #82 #99 #100 decided 2026-11-14; no override author while agents post as the owner (#78) self.assertEqual((r.mode, r.findings, r.rounds.approach_at, r.rounds.max, r.warnings, r.override_authors), - ('shadow', True, 3, 4, (), ())) - self.assertEqual(r.order, ()) # enforce would take it as it is + ('enforce', True, 3, 4, (), ())) + self.assertEqual(r.order, ()) self.assertGreaterEqual(r.min_lenses.critical, r.min_lenses.normal + 1) # #82: one extra review + self.assertEqual(r.rounds.approach_lens, 'approach') for x in r.lenses: with self.subTest(x.name): self.assertTrue(os.path.isfile(os.path.join(ROOT, x.prompt)), x.prompt) def test_workflow_queue_gate_config_and_lens_files_are_critical(self): # #82 - critical = next(x for x in self.review.rules if x.weight == 'critical') for path in ('.github/workflows/fork-head-gate.yml', 'fork-ci/actions/pr-queue/review_judge.py', 'fork-ci/actions/head-check/head_check.py', FORK_CONFIG, '.github/fork-lens-state.md', - '.github/actions/setup-ci-deps/action.yml', '.github/fork-ci-release-rulesets.json'): + '.github/actions/setup-ci-deps/action.yml', '.github/fork-ci-release-rulesets.json', + '.github/fork-actions-policies.json', '.github/scripts/fork-contracts.test.py', + '.github/workflows/new.yml', '.github/new-folder/x.json', '.github/new.md', 'fork-ci/new.txt', + 'fork-kit/review/lib/review_kit.py', 'fork-kit/gh-apps/libexec/x', 'fork-kit/review/SKILL.md', + 'docs/fork/gate-trust.md'): with self.subTest(path): - j = rj.judge(self.review, [('M', (path,))], FORK_CONFIG) - self.assertEqual((j.weight, j.lenses), ('critical', critical.lenses)) + j = self.judge(path) + self.assertEqual((j.weight, j.lenses), ('critical', self.critical.lenses)) self.assertGreaterEqual(len(j.lenses), 2) - j = rj.judge(self.review, [('M', ('apps/server/src/x.ts',))], FORK_CONFIG) - self.assertEqual(j.weight, 'normal') - self.assertTrue(j.lenses) + j = self.judge('apps/server/src/x.ts') + self.assertEqual((j.weight, j.lenses), ('normal', ('code',))) + + def test_every_github_file_is_critical_but_the_two_bookkeeping_lists(self): # a new one must be sorted + top = sorted(os.listdir(os.path.join(ROOT, '.github'))) + self.assertTrue(set(NOT_GATE) <= {f'.github/{x}' for x in top}, 'a bookkeeping list moved: update NOT_GATE') + for name in top: + path = f'.github/{name}' + ('/x' if os.path.isdir(os.path.join(ROOT, '.github', name)) else '') + with self.subTest(path): + weight = self.judge(path).weight + if path in NOT_GATE: + self.assertEqual(weight, 'normal', f'{path} is a bookkeeping list (NOT_GATE)') + else: + self.assertEqual(weight, 'critical', f'{path} is not in the gate rule: add it to the gate rule of ' + f'{FORK_CONFIG} (a workflow file or a gate setting, #82), or to NOT_GATE with why') + + def test_only_the_gate_trust_page_of_docs_fork_is_critical(self): + # docs/fork/gate-trust.md holds the owner's paste commands that set up the gate App's key and tokens (a gate + # setting, like fork-kit/gh-apps); the other docs/fork pages are runbooks and stay normal, with the code lens, + # never a no-lens rule (a changed owner command would merge unread). + self.assertEqual(self.judge('docs/fork/gate-trust.md').hits[0].rule, self.critical.name) + for name in sorted(os.listdir(os.path.join(ROOT, 'docs', 'fork'))): + if name == 'gate-trust.md': + continue + with self.subTest(name): + j = self.judge(f'docs/fork/{name}') + self.assertEqual((j.weight, j.lenses), ('normal', ('code',))) + j = rj.judge(self.review, [('M', ('docs/fork/gate-trust.md',)), ('M', ('docs/fork/host-tools.md',))], + FORK_CONFIG) + self.assertEqual((j.weight, j.lenses), ('critical', self.critical.lenses + ('code',))) + + def test_every_fork_kit_folder_is_sorted(self): # a new kit must be sorted too + top = sorted(os.listdir(os.path.join(ROOT, 'fork-kit'))) + self.assertTrue(set(GATE_KITS) <= set(top), 'a gate kit moved: update GATE_KITS and the gate rule') + for name in top: + path = f'fork-kit/{name}' + ('/x' if os.path.isdir(os.path.join(ROOT, 'fork-kit', name)) else '') + with self.subTest(path): + weight = self.judge(path).weight + if name in GATE_KITS: + self.assertEqual(weight, 'critical', f'{path} is a gate kit (GATE_KITS): add it to the gate rule') + else: + self.assertIn(name, OTHER_KITS, f'{path} is new: add it to the gate rule of {FORK_CONFIG} and ' + 'GATE_KITS (it decides what the gate checks or holds its keys, #82), or to ' + 'OTHER_KITS with why') + self.assertEqual(weight, 'normal', f'{path} is in OTHER_KITS but judged {weight}') + + def test_an_app_change_that_announces_a_seam_or_lists_a_test_needs_the_code_lens(self): # the dry run's refusals + for extra in NOT_GATE: + with self.subTest(extra): + j = rj.judge(self.review, [('M', ('apps/server/src/x.ts',)), ('M', (extra,))], FORK_CONFIG) + self.assertEqual((j.weight, j.lenses), ('normal', ('code',))) + + def test_an_app_change_plus_a_gate_file_needs_the_code_lens_and_the_gate_lenses(self): # #82: one EXTRA review + for gate in ('.github/workflows/ci.yml', '.github/SECURITY.md', 'fork-ci/actions/x.py', + 'fork-kit/review/lib/review_kit.py'): + with self.subTest(gate): + j = rj.judge(self.review, [('M', ('apps/server/src/x.ts',)), ('M', ('.github/fork-seams.json',)), + ('M', (gate,))], FORK_CONFIG) + self.assertEqual((j.weight, j.lenses), ('critical', ('merge-path', 'state', 'code'))) + self.assertEqual(self.judge('.github/workflows/ci.yml').lenses, ('merge-path', 'state')) # a gate change alone if __name__ == '__main__': diff --git a/fork-kit/review/tests/plan.test.py b/fork-kit/review/tests/plan.test.py index 2f39ade25ba31..3589548f8fe11 100644 --- a/fork-kit/review/tests/plan.test.py +++ b/fork-kit/review/tests/plan.test.py @@ -96,8 +96,11 @@ def test_a_critical_path_needs_two_independent_lenses(self): code, out, err = self.plan(changes(('M', 'src/storage/wallet.ts'), ('M', 'src/app.ts'))) self.assertEqual((code, err), (0, '')) self.assertIn('weight: critical\n', out) - self.assertIn('lenses: data-1, data-2 (independent: one new read-only agent each, no shared context', out) + # every rule hit gives its lenses: the two critical reads, and the app file's own `code` read (issue 82) + self.assertIn('lenses: data-1, data-2, code (independent: one new read-only agent each, no shared context', out) self.assertIn(' data-1: prompt .github/lenses/data.md; runner claude-opus-5-5 high\n', out) + self.assertIn(' data-2: prompt .github/lenses/data.md; runner claude-opus-5-5 high\n', out) + self.assertIn(' code: prompt .github/lenses/code.md; runner claude-opus-5-5 high\n', out) self.assertIn('note: the review block is in shadow', out) def test_a_normal_path_needs_one_lens(self):