Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions .github/fork-pr-queue.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,32 @@
"head_check": "head_check",
"review": {
"version": 1,
"mode": "shadow",
"mode": "enforce",
"rules": [
{
"name": "gate",
"weight": "critical",
"paths": [".github/**", "fork-ci/**"],
"paths": [
".github/*/*/**",
".github/*.md",
".github/*.td",
".github/*.yml",
".github/*.yaml",
".github/fork-actions-policies.json",
".github/fork-ci-release*.json",
".github/fork-ci-rerun.json",
".github/fork-fleet.json",
".github/fork-owned-paths.txt",
".github/fork-pr-queue.json",
".github/fork-removed-workflows.txt",
".github/fork-secret-sweep-gone-tips.txt",
".github/fork-tests-upstream.json",
".github/fork-upstream.json",
"docs/fork/gate-trust.md",
"fork-ci/**",
"fork-kit/gh-apps/**",
"fork-kit/review/**"
],
"lenses": ["merge-path", "state"]
},
{ "name": "code", "weight": "normal", "paths": ["**"], "lenses": ["code"] }
Expand All @@ -34,6 +54,7 @@
"min_lenses": { "critical": 2, "normal": 1, "light": 0 },
"rounds": { "approach_at": 3, "max": 4, "approach_lens": "approach" },
"findings": true,
"override_authors": [],
"state_line": false
}
}
7 changes: 4 additions & 3 deletions apps/server/src/fork/reviewRules/plan.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ export interface ReviewPlan {
/**
* The plan for `changes` as `git diff --name-status -M` names them ([status, path] or [R|C<score>, old, new]) under a
* valid `review` block; `configPath` is the config file (repo-relative). A rename or copy counts under both paths; a
* protected path also counts for the first critical rule; the heaviest weight hit decides; no change at all counts as
* the catch-all rule.
* protected path also counts for the first critical rule; the heaviest weight hit is the weight; the lenses are those
* of every rule hit, in rule order, each once; no change at all counts as the catch-all rule.
*/
export function planReview(
review: ReviewBlock,
Expand Down Expand Up @@ -165,8 +165,9 @@ export function planReview(
: [];
});
const weight = WEIGHTS.find((w) => hits.some((h) => h.weight === w))!;
// Every rule hit gives its lenses, whatever its weight (owner decision: fork issue 82, one extra review).
const lenses = review.rules
.filter((r) => r.weight === weight && hits.some((h) => h.rule === r.name))
.filter((r) => hits.some((h) => h.rule === r.name))
.flatMap((r) => r.lenses);
return { weight, lenses: [...new Set(lenses)], hits, protected: isProtected };
}
14 changes: 14 additions & 0 deletions fork-ci/RELEASE-NOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,20 @@ nothing.

## After ci/v14

### pr-queue and head-check: every rule hit gives its lenses (#82)

With a `review` block in `enforce`, the required lenses are now the lenses of every rule a changed path counted for,
whatever its weight, not only those of the rules of the heaviest weight hit. An app change plus one critical file needs
the app rule's lens and the critical lenses; before, the critical lenses alone passed it and no lens read the app code.
The weight is still the heaviest hit. T3's copy of the rules (`apps/server/src/fork/reviewRules`) plans the same lenses.
First in the release after `ci/v22`.

**Upgrade:** a config in `enforce` needs more lens reads on mixed changes after the move, and a built-in protected path
(the config, a lens prompt, a workflow) under a lighter rule also needs that rule's lenses. To keep the protected paths
at the critical lenses only, put them in a critical rule. In `shadow` only the summary row changes. `ci/v21` and `ci/v22`
accept the same config but judge mixed changes the old, looser way, so a caller of a config in `enforce` pins this release
or a later one (pr-queue README, "Turning it on: release, pin, then config", step 1).

### pr-queue and head-check: review findings, rounds and `override_authors` (#740)

With a `review` block, the gate also judges the review itself from the lens comments. With `findings: true`, a lens's
Expand Down
7 changes: 4 additions & 3 deletions fork-ci/actions/head-check/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,10 @@ All of them are the queue's, imported from `../pr-queue`, never copied:
- **Lenses by changed path** (`review_judge.py`, the pr-queue README, "Review
rules"): when the base's config has a `review` block, the paths the head
changes against its merge base (`git diff --name-status -M`, renames under
both paths, no cap) are judged by its rules. In `enforce` the lenses of the
heaviest weight hit are required on top of `required-lenses`; in `shadow` the
summary only adds a row "the path rules would have said: ...". A change the
both paths, no cap) are judged by its rules. In `enforce` the lenses of every
rule hit (whatever its weight) are required on top of `required-lenses`; in
`shadow` the summary only adds a row "the path rules would have said: ...".
A change the
rules cannot judge (no merge base, a failed diff) refuses, in either mode.
Without the block, nothing of this runs.
- **Findings and rounds** (`review_judge.judge_rounds`, the pr-queue README,
Expand Down
185 changes: 173 additions & 12 deletions fork-ci/actions/head-check/head_check.test.py
Original file line number Diff line number Diff line change
Expand Up @@ -695,15 +695,18 @@ def test_shadow_only_says_what_the_rules_would_have_needed(self):
self.assertEqual(code, 1, out)
self.assertIn('required lens without a read: design', out)

def test_enforce_needs_the_lenses_of_the_heaviest_rule_hit(self):
def test_enforce_needs_the_lenses_of_every_rule_hit(self): # #82: the critical lenses never replace code's
base = self.base()
head = self.head(base, **{'src/storage/s.py': lines(40, ['x'], 3), 'src/app.py': 'a\n', 'docs/d.md': 'd\n'})
code, out = self.check(base, head, ['code'])
self.assertEqual(code, 1, out)
self.assertIn('required lens without a read: data-1, data-2', out)
self.assertIn('path rules (enforce): the path rules decide: critical, lenses data-1, data-2 (3 changed paths',
out)
self.assertEqual(self.check(base, head, ['data-1', 'data-2'])[0], 0)
self.assertIn('path rules (enforce): the path rules decide: critical, lenses data-1, data-2, code (3 changed '
'paths', out)
code, out = self.check(base, head, ['data-1', 'data-2'])
self.assertEqual(code, 1, out)
self.assertIn('required lens without a read: code', out)
self.assertEqual(self.check(base, head, ['data-1', 'data-2', 'code'])[0], 0)

def test_enforce_still_requires_the_input_lenses(self):
base = self.base()
Expand Down Expand Up @@ -748,24 +751,28 @@ def test_a_rename_counts_under_its_old_and_its_new_path(self):
self.assertEqual(code, 1, out)
self.assertIn('data-1, data-2', out)
head = self.head(base, drop=['src/storage/s.py'], **{'src/s.py': lines(40)})
code, out = self.check(base, head, ['data-1', 'data-2'])
code, out = self.check(base, head, ['data-1', 'data-2', 'code']) # the new path is code's to read (#82)
self.assertEqual(code, 0, out)
self.assertIn('data (critical): src/storage/s.py; code (normal): src/s.py', out)

def test_the_config_and_the_lens_prompts_are_always_critical(self):
base = self.base()
prompt = REVIEW_BLOCK['lenses']['code']['prompt']
for name, kw in (('config edited', {'cfg': review_cfg('enforce', findings=False)}), # deleted: refused (#388)
('prompt renamed', {'drop': [prompt], 'docs/code-lens.md': PROMPTS[prompt]}),
('prompt edited', {prompt: 'changed\n'}),
('workflow', {'.github/workflows/ci.yml': 'on: push\n'})):
with self.subTest(name):
# A protected path also counts for its own rule: under the catch-all (the config, a workflow) it needs the
# code lens too, every rule hit gives its lenses (#82); a prompt under the light docs rule adds none.
for name, kw, own in (('config edited', {'cfg': review_cfg('enforce', findings=False)}, ['code']),
('prompt renamed', {'drop': [prompt], 'docs/code-lens.md': PROMPTS[prompt]}, []),
('prompt edited', {prompt: 'changed\n'}, []),
('workflow', {'.github/workflows/ci.yml': 'on: push\n'}, ['code'])):
with self.subTest(name): # the config deleted: refused (#388)
head = self.head(base, **kw)
code, out = self.check(base, head, ['code'])
self.assertEqual(code, 1, out)
self.assertIn('critical, lenses data-1, data-2', out)
self.assertIn('critical, lenses ' + ', '.join(['data-1', 'data-2', *own]) + ' (', out)
self.assertIn('protected: ', out)
self.assertEqual(self.check(base, head, ['data-1', 'data-2'])[0], 0)
self.assertEqual(self.check(base, head, ['data-1', 'data-2', *own])[0], 0)
if own:
self.assertEqual(self.check(base, head, ['data-1', 'data-2'])[0], 1)

def test_an_empty_diff_is_judged_by_the_catch_all_rule(self): # empty state: never "no lens needed"
base = self.base()
Expand Down Expand Up @@ -1538,6 +1545,160 @@ class FindingsAndRoundsBare(FindingsAndRounds):
BARE = True


# --- the fork's own config: .github/fork-pr-queue.json in enforce (#82, #99, #100; no override author, #78) ---------
FORK_ROOT = os.path.join(HERE, '..', '..', '..')
FORK_CONFIG_PATH = '.github/fork-pr-queue.json'
with open(os.path.join(FORK_ROOT, FORK_CONFIG_PATH), encoding='utf-8') as _f:
FORK_RAW = json.load(_f)


def fork_prompts():
out = {}
for x in FORK_RAW['review']['lenses'].values():
with open(os.path.join(FORK_ROOT, x['prompt']), encoding='utf-8') as f:
out[x['prompt']] = f.read()
return out


class ForkConfigRules(unittest.TestCase):
"""The gate on a base that holds the fork's real config and lens prompts: each rule the owner decided blocks when it
should and passes when it should, and the same config in shadow only reports. A test that fails here after a
config change means the change alters what the fork's gate decides."""
BARE = False
FILES = {'apps/server/src/x.ts': lines(), '.github/workflows/ci.yml': lines(), 'fork-ci/actions/x.py': lines(),
'.github/fork-seams.json': lines(), '.github/fork-ci-tests.txt': lines()}

def setUp(self):
self.w = World(self.BARE)
self.addCleanup(self.w.close)
self.p = Posts()

def pair(self, *changed, mode=None):
"""A base with the fork's config (its mode replaced by `mode`) and a head on it that changes `changed`."""
raw = json.loads(json.dumps(FORK_RAW))
if mode:
raw['review']['mode'] = mode
tree = {FORK_CONFIG_PATH: json.dumps(raw, indent=2) + '\n', **fork_prompts(), **self.FILES}
base = self.w.commit(tree)
return base, self.w.commit({**tree, **{p: lines(30, ['x'], 3) for p in changed or ('apps/server/src/x.ts',)}},
[base])

def check(self, base, head, comments):
path = self.w.checkout(base, head)
return run_cli(path, fake_gh((0, json.dumps(pr(comments, head=head)))), config_path=FORK_CONFIG_PATH)

def said(self, name, sha, *lines):
return FindingsAndRounds.said(self, name, sha, *lines)

def override(self, sha): # from the owner's login, which agents post from too (#78)
return self.p.post(f"{FORK_RAW['override']['prefix']} {sha}: read it myself")

def test_it_is_in_enforce(self):
self.assertEqual(FORK_RAW['review']['mode'], 'enforce')
# #78: no person posts from a login of their own yet. Written out, empty: releases ci/v20 and older do not know
# the key and refuse the config (fail closed), where they would judge no finding and no round (fail open).
self.assertEqual(FORK_RAW['review']['override_authors'], [])

def test_a_workflow_or_gate_change_needs_the_extra_review(self): # #82
for changed in ('.github/workflows/ci.yml', 'fork-ci/actions/x.py', 'docs/fork/gate-trust.md'):
with self.subTest(changed):
base, head = self.pair(changed)
code, out = self.check(base, head, [self.said('code', head)])
self.assertEqual(code, 1, out)
self.assertIn('required lens without a read: merge-path, state', out)
code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)])
self.assertEqual(code, 0, out)
code, out = self.check(base, head, [self.said('merge-path', head)]) # one review is not enough
self.assertEqual(code, 1, out)

def test_an_app_change_plus_a_gate_file_needs_the_code_lens_too(self): # #82: one EXTRA review, not instead
for gate in ('.github/workflows/ci.yml', 'fork-ci/actions/x.py'):
with self.subTest(gate):
base, head = self.pair('apps/server/src/x.ts', gate)
code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)])
self.assertEqual(code, 1, out) # nobody read the app code
self.assertIn('required lens without a read: code', out)
self.assertIn('the path rules decide: critical, lenses merge-path, state, code (', out)
code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head),
self.said('code', head)])
self.assertEqual(code, 0, out)

def test_the_rollback_to_shadow_needs_the_gate_lenses(self): # it changes the config: judged by the base's rules
raw = json.loads(json.dumps(FORK_RAW))
raw['review']['mode'] = 'shadow'
base, _ = self.pair()
tree = {FORK_CONFIG_PATH: json.dumps(FORK_RAW, indent=2) + '\n', **fork_prompts(), **self.FILES}
head = self.w.commit({**tree, FORK_CONFIG_PATH: json.dumps(raw, indent=2) + '\n'}, [base])
code, out = self.check(base, head, [self.said('code', head)])
self.assertEqual(code, 1, out)
self.assertIn('required lens without a read: merge-path, state', out)
code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)])
self.assertEqual(code, 0, out)

def test_an_app_change_that_adds_a_seam_or_a_test_line_needs_the_code_lens(self): # the dry run's wrong refusals
for extra in ('.github/fork-seams.json', '.github/fork-ci-tests.txt'):
with self.subTest(extra):
base, head = self.pair('apps/server/src/x.ts', extra)
code, out = self.check(base, head, [self.said('code', head)])
self.assertEqual(code, 0, out)
self.assertIn('path rules (enforce): the path rules decide: normal, lenses code (', out)
code, out = self.check(base, head, [self.said('merge-path', head), self.said('state', head)])
self.assertEqual(code, 1, out) # the gate lenses alone never read app code
self.assertIn('required lens without a read: code', out)

def test_a_blocker_or_major_fails_and_the_owners_login_does_not_clear_it(self): # #99, #78
base, head = self.pair()
self.assertEqual(self.check(base, head, [self.said('code', head)])[0], 0)
for line in ('major: two writers race', 'blocker: data lost on restart'):
with self.subTest(line):
code, out = self.check(base, head, [self.said('code', head, line)])
self.assertEqual(code, 1, out)
self.assertIn(f"that no later round cleared (issue #99): '{line}'", out)
code, out = self.check(base, head, [self.said('code', head, line), self.override(head)])
self.assertEqual(code, 1, out)
code, out = self.check(base, head, [self.said('code', head, 'Looks good to me.')]) # no findings section
self.assertEqual(code, 1, out)
self.assertIn('has no findings section', out)

def test_round_three_needs_the_approach_lens_and_round_five_is_past_the_cap(self): # #100
base, head = self.pair()
two = [self.said('code', OLD[0]), self.said('code r2', head)]
self.assertEqual(self.check(base, head, two)[0], 0)
three = [self.said('code', OLD[0]), self.said('code r2', OLD[1]), self.said('code r3', head)]
code, out = self.check(base, head, three)
self.assertEqual(code, 1, out)
self.assertIn('required lens without a read: approach', out)
code, out = self.check(base, head, three + [self.said('approach', head)])
self.assertEqual(code, 0, out)
self.assertIn('round 3 of 4, approach lens approach required from round 3', out)
five = [self.said('code', sha) for sha in OLD] + [self.said('code', head), self.said('approach', head)]
code, out = self.check(base, head, five)
self.assertEqual(code, 1, out)
self.assertIn('review round 5 is past the cap of 4 rounds (issue #100)', out)
code, out = self.check(base, head, five + [self.override(head)]) # no override author: no person decides
self.assertEqual(code, 1, out)
self.assertIn('no one is listed in review.override_authors yet, so no override is accepted', out)

def test_shadow_only_reports_what_enforce_refuses(self): # the rollback: the same config in shadow
base, head = self.pair('.github/workflows/ci.yml', mode='shadow')
code, out = self.check(base, head, [self.said('code', head)])
self.assertEqual(code, 0, out)
self.assertIn('path rules (shadow): the path rules would have said: critical, lenses merge-path, state', out)
base, head = self.pair(mode='shadow')
code, out = self.check(base, head, [self.said('code', head, 'major: two writers race')])
self.assertEqual(code, 0, out)
self.assertIn('findings and rounds (shadow): they would have said: round 1 of 4; lens code reported a major',
out)
five = [self.said('code', sha) for sha in OLD] + [self.said('code', head)]
code, out = self.check(base, head, five)
self.assertEqual(code, 0, out)
self.assertIn('findings and rounds (shadow): they would have said: round 5 of 4', out)


class ForkConfigRulesBare(ForkConfigRules):
BARE = True


class FetchTimeout(unittest.TestCase): # #740 round 2, B minor 4: a hung fetch of a read commit never hangs the check
def test_a_fetch_that_takes_too_long_failed(self):
timeouts = []
Expand Down
Loading
Loading