diff --git a/.github/workflows/audit-central-ruleset.yml b/.github/workflows/audit-central-ruleset.yml index ee93de9b0..1486f4162 100644 --- a/.github/workflows/audit-central-ruleset.yml +++ b/.github/workflows/audit-central-ruleset.yml @@ -41,6 +41,7 @@ jobs: ORG_LOGIN: ContextualWisdomLab RULESET_ID: "18156473" STACKED_RULESET_ID: "21732164" + REPOSITORY_RULESET_ID: "17921150" RULESET_SENTINEL_REPOSITORY: naruon run: | set -euo pipefail @@ -91,6 +92,16 @@ jobs: "$ruleset_json" >"$ruleset_with_scope_json" python3 scripts/ci/audit_central_required_workflows.py "$ruleset_with_scope_json" + repository_ruleset_json="$RUNNER_TEMP/owner-repository-ruleset.json" + repository_ruleset_error="$RUNNER_TEMP/owner-repository-ruleset.error" + repository_ruleset_endpoint="repos/${ORG_LOGIN}/.github/rulesets/${REPOSITORY_RULESET_ID}?includes_parents=true" + if ! gh api "$repository_ruleset_endpoint" >"$repository_ruleset_json" 2>"$repository_ruleset_error"; then + echo "::error::Ruleset audit could not read owner repository ruleset ${REPOSITORY_RULESET_ID}." + sed 's/^/ /' "$repository_ruleset_error" + exit 1 + fi + python3 scripts/ci/audit_central_required_workflows.py --repository "$repository_ruleset_json" + stacked_ruleset_json="$RUNNER_TEMP/stacked-opencode-ruleset.json" stacked_ruleset_error="$RUNNER_TEMP/stacked-opencode-ruleset.error" stacked_endpoint="repos/${ORG_LOGIN}/${RULESET_SENTINEL_REPOSITORY}/rulesets/${STACKED_RULESET_ID}?includes_parents=true" diff --git a/scripts/ci/audit_central_required_workflows.py b/scripts/ci/audit_central_required_workflows.py index 4aa33929c..0eeaf87c6 100644 --- a/scripts/ci/audit_central_required_workflows.py +++ b/scripts/ci/audit_central_required_workflows.py @@ -5,15 +5,17 @@ import argparse import json -from pathlib import Path import sys +from pathlib import Path from typing import Any, TextIO - RULESET_ID = 18156473 RULESET_NAME = "CWL Central required workflows" STACKED_RULESET_ID = 21732164 STACKED_RULESET_NAME = "CWL Stacked OpenCode required workflow" +REPOSITORY_RULESET_ID = 17921150 +REPOSITORY_RULESET_NAME = "Lock default branch" +REPOSITORY_RULESET_SOURCE = "ContextualWisdomLab/.github" SOURCE_REPOSITORY_ID = 1274066402 SOURCE_REF = "refs/heads/main" SOURCE_ORGANIZATION = "ContextualWisdomLab" @@ -59,6 +61,8 @@ def audit_ruleset(payload: dict[str, Any]) -> list[str]: errors.append("central ruleset target is not branch") if payload.get("enforcement") != "active": errors.append("central ruleset enforcement is not active") + if payload.get("bypass_actors") != []: + errors.append("central ruleset must not configure bypass actors") conditions = payload.get("conditions") conditions = conditions if isinstance(conditions, dict) else {} @@ -115,25 +119,47 @@ def audit_ruleset(payload: dict[str, Any]) -> list[str]: ref_names = conditions.get("ref_name") ref_names = ref_names if isinstance(ref_names, dict) else {} - if "~DEFAULT_BRANCH" not in (ref_names.get("include") or []): - errors.append("central ruleset does not target every default branch") + if ( + ref_names.get("include") != ["~DEFAULT_BRANCH"] + or ref_names.get("exclude") != [] + ): + errors.append("central ruleset ref scope must be exactly the default branch") workflow_rules = _typed_rules(payload, "workflows") + workflow_parameters: dict[str, Any] = {} if len(workflow_rules) != 1: errors.append(f"expected one workflows rule, found {len(workflow_rules)}") workflows: list[Any] = [] else: parameters = workflow_rules[0].get("parameters") - parameters = parameters if isinstance(parameters, dict) else {} - workflows = parameters.get("workflows") + workflow_parameters = parameters if isinstance(parameters, dict) else {} + workflows = workflow_parameters.get("workflows") workflows = workflows if isinstance(workflows, list) else [] + if len(workflow_rules) == 1 and workflow_parameters.get("do_not_enforce_on_create") is not True: + errors.append("central required workflows block the branch create transition") + + malformed_workflows = sum( + 1 + for workflow in workflows + if not isinstance(workflow, dict) or not isinstance(workflow.get("path"), str) + ) + if malformed_workflows: + suffix = "entry" if malformed_workflows == 1 else "entries" + errors.append( + f"central required workflows contain {malformed_workflows} malformed {suffix}" + ) + workflows_by_path: dict[str, list[dict[str, Any]]] = {} for workflow in workflows: if not isinstance(workflow, dict) or not isinstance(workflow.get("path"), str): continue workflows_by_path.setdefault(workflow["path"], []).append(workflow) + unexpected_workflows = sorted(set(workflows_by_path) - set(REQUIRED_WORKFLOW_PATHS)) + if unexpected_workflows: + errors.append(f"unexpected central required workflows: {unexpected_workflows}") + for path in REQUIRED_WORKFLOW_PATHS: matches = workflows_by_path.get(path, []) if not matches: @@ -167,8 +193,8 @@ def audit_ruleset(payload: dict[str, Any]) -> list[str]: if parameters.get("required_review_thread_resolution") is not True: errors.append("review-thread resolution protection is disabled") allowed_methods = set(parameters.get("allowed_merge_methods") or []) - if not {"merge", "squash"}.issubset(allowed_methods): - errors.append("merge and squash are not both allowed merge methods") + if allowed_methods != {"merge", "squash"}: + errors.append("only merge and squash may be allowed merge methods") if not _typed_rules(payload, "deletion"): errors.append("default-branch deletion protection is missing") @@ -237,6 +263,55 @@ def audit_stacked_ruleset(payload: dict[str, Any]) -> list[str]: return errors +def audit_repository_ruleset(payload: dict[str, Any]) -> list[str]: + """Return drift reasons for the owner repository's default-branch policy.""" + + errors: list[str] = [] + if payload.get("id") != REPOSITORY_RULESET_ID: + errors.append(f"expected repository ruleset id {REPOSITORY_RULESET_ID}") + if payload.get("name") != REPOSITORY_RULESET_NAME: + errors.append(f"expected repository ruleset name {REPOSITORY_RULESET_NAME}") + if payload.get("source_type") != "Repository" or payload.get("source") != REPOSITORY_RULESET_SOURCE: + errors.append("repository ruleset source is not ContextualWisdomLab/.github") + if payload.get("target") != "branch": + errors.append("repository ruleset target is not branch") + if payload.get("enforcement") != "active": + errors.append("repository ruleset enforcement is not active") + if payload.get("bypass_actors") != []: + errors.append("repository ruleset must not configure bypass actors") + + conditions = payload.get("conditions") + conditions = conditions if isinstance(conditions, dict) else {} + ref_names = conditions.get("ref_name") + ref_names = ref_names if isinstance(ref_names, dict) else {} + if ref_names != {"include": ["~DEFAULT_BRANCH"], "exclude": []}: + errors.append("repository ruleset ref scope must be exactly the default branch") + + review_rules = _typed_rules(payload, "pull_request") + if len(review_rules) != 1: + errors.append(f"expected one repository pull_request rule, found {len(review_rules)}") + else: + raw_parameters = review_rules[0].get("parameters") + parameters = raw_parameters if isinstance(raw_parameters, dict) else {} + if parameters.get("required_approving_review_count") != 2: + errors.append("repository ruleset does not require exactly two approving reviews") + if parameters.get("dismiss_stale_reviews_on_push") is not True: + errors.append("repository ruleset stale-review dismissal on push is disabled") + if parameters.get("require_last_push_approval") is not True: + errors.append("repository ruleset last-push approval protection is disabled") + if parameters.get("required_review_thread_resolution") is not True: + errors.append("repository ruleset review-thread resolution protection is disabled") + allowed_methods = set(parameters.get("allowed_merge_methods") or []) + if allowed_methods != {"merge", "squash"}: + errors.append("repository ruleset must allow only merge and squash") + + if not _typed_rules(payload, "deletion"): + errors.append("repository default-branch deletion protection is missing") + if not _typed_rules(payload, "non_fast_forward"): + errors.append("repository default-branch non-fast-forward protection is missing") + return errors + + def load_payload(path: Path | None, stdin: TextIO) -> dict[str, Any]: """Load a ruleset object from ``path`` or standard input.""" if path is None: @@ -252,7 +327,9 @@ def load_payload(path: Path | None, stdin: TextIO) -> dict[str, Any]: def parse_args(argv: list[str] | None = None) -> argparse.Namespace: """Parse the optional ruleset JSON path.""" parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--stacked", action="store_true") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--stacked", action="store_true") + mode.add_argument("--repository", action="store_true") parser.add_argument("ruleset_json", nargs="?", type=Path) return parser.parse_args(argv) @@ -266,9 +343,18 @@ def main(argv: list[str] | None = None) -> int: print(f"ERROR: unable to load ruleset JSON: {exc}", file=sys.stderr) return 2 - auditor = audit_stacked_ruleset if args.stacked else audit_ruleset - ruleset_id = STACKED_RULESET_ID if args.stacked else RULESET_ID - workflow_count = 1 if args.stacked else len(REQUIRED_WORKFLOW_PATHS) + if args.repository: + auditor = audit_repository_ruleset + ruleset_id = REPOSITORY_RULESET_ID + workflow_count = 0 + elif args.stacked: + auditor = audit_stacked_ruleset + ruleset_id = STACKED_RULESET_ID + workflow_count = 1 + else: + auditor = audit_ruleset + ruleset_id = RULESET_ID + workflow_count = len(REQUIRED_WORKFLOW_PATHS) errors = auditor(payload) if errors: for error in errors: @@ -279,7 +365,9 @@ def main(argv: list[str] | None = None) -> int: ) return 1 - if args.stacked: + if args.repository: + print(f"PASS: repository ruleset {ruleset_id} protects the default branch") + elif args.stacked: print( f"PASS: ruleset {ruleset_id} audits {workflow_count} " "central required workflows in evaluate mode" diff --git a/tests/test_central_required_workflow_ruleset_audit.py b/tests/test_central_required_workflow_ruleset_audit.py index 00d28288a..35b050eb8 100644 --- a/tests/test_central_required_workflow_ruleset_audit.py +++ b/tests/test_central_required_workflow_ruleset_audit.py @@ -1,12 +1,13 @@ +import json from copy import deepcopy from io import StringIO -import json from pathlib import Path from scripts.ci import audit_central_required_workflows as audit - REPO_ROOT = Path(__file__).resolve().parents[1] + + def ruleset_payload() -> dict: """Return the expected live central required-workflow ruleset shape.""" workflow_paths = ( @@ -23,6 +24,7 @@ def ruleset_payload() -> dict: "name": "CWL Central required workflows", "target": "branch", "enforcement": "active", + "bypass_actors": [], "conditions": { "repository_name": { "include": ["~ALL"], @@ -34,7 +36,7 @@ def ruleset_payload() -> dict: { "type": "workflows", "parameters": { - "do_not_enforce_on_create": False, + "do_not_enforce_on_create": True, "workflows": [ { "repository_id": 1274066402, @@ -108,6 +110,37 @@ def stacked_ruleset_payload() -> dict: } +def repository_ruleset_payload() -> dict: + """Return the expected strong default-branch policy for the owner repo.""" + + return { + "id": 17921150, + "name": "Lock default branch", + "target": "branch", + "source_type": "Repository", + "source": "ContextualWisdomLab/.github", + "enforcement": "active", + "bypass_actors": [], + "conditions": { + "ref_name": {"include": ["~DEFAULT_BRANCH"], "exclude": []}, + }, + "rules": [ + { + "type": "pull_request", + "parameters": { + "required_approving_review_count": 2, + "dismiss_stale_reviews_on_push": True, + "require_last_push_approval": True, + "required_review_thread_resolution": True, + "allowed_merge_methods": ["merge", "squash"], + }, + }, + {"type": "deletion"}, + {"type": "non_fast_forward"}, + ], + } + + def test_expected_central_ruleset_passes(monkeypatch, capsys) -> None: monkeypatch.setattr(audit.sys, "stdin", StringIO(json.dumps(ruleset_payload()))) @@ -118,10 +151,226 @@ def test_expected_central_ruleset_passes(monkeypatch, capsys) -> None: ) +def test_central_ruleset_rejects_unexpected_and_malformed_workflows() -> None: + payload = ruleset_payload() + workflow_rule = next(rule for rule in payload["rules"] if rule["type"] == "workflows") + workflow_rule["parameters"]["workflows"].extend( + [ + { + "repository_id": 1274066402, + "path": ".github/workflows/unexpected.yml", + "ref": "refs/heads/main", + }, + {"repository_id": 1274066402, "path": 42, "ref": "refs/heads/main"}, + ] + ) + + errors = audit.audit_ruleset(payload) + + assert "unexpected central required workflows: ['.github/workflows/unexpected.yml']" in errors + assert "central required workflows contain 1 malformed entry" in errors + + +def test_central_ruleset_rejects_rebase_merge_method() -> None: + payload = ruleset_payload() + review_rule = next(rule for rule in payload["rules"] if rule["type"] == "pull_request") + review_rule["parameters"]["allowed_merge_methods"].append("rebase") + + assert "only merge and squash may be allowed merge methods" in audit.audit_ruleset(payload) + + +def test_central_ruleset_rejects_bypass_actors() -> None: + payload = ruleset_payload() + payload["bypass_actors"] = [ + { + "actor_id": None, + "actor_type": "OrganizationAdmin", + "bypass_mode": "always", + } + ] + + assert audit.audit_ruleset(payload) == [ + "central ruleset must not configure bypass actors", + ] + + +def test_central_ruleset_rejects_missing_bypass_evidence() -> None: + payload = ruleset_payload() + del payload["bypass_actors"] + + assert audit.audit_ruleset(payload) == [ + "central ruleset must not configure bypass actors", + ] + + def test_inherited_ruleset_and_organization_scope_probes_pass() -> None: assert audit.audit_ruleset(inherited_ruleset_payload()) == [] +def test_expected_repository_ruleset_passes() -> None: + assert hasattr(audit, "audit_repository_ruleset"), ( + "the central audit must inspect the repository ruleset that protects .github" + ) + assert audit.audit_repository_ruleset(repository_ruleset_payload()) == [] + + +def test_repository_ruleset_rejects_live_weakened_review_controls() -> None: + assert hasattr(audit, "audit_repository_ruleset"), ( + "the central audit must inspect the repository ruleset that protects .github" + ) + payload = repository_ruleset_payload() + review_rule = next(rule for rule in payload["rules"] if rule["type"] == "pull_request") + review_rule["parameters"]["required_approving_review_count"] = 0 + review_rule["parameters"]["require_last_push_approval"] = False + + assert audit.audit_repository_ruleset(payload) == [ + "repository ruleset does not require exactly two approving reviews", + "repository ruleset last-push approval protection is disabled", + ] + + +def test_repository_ruleset_rejects_rebase_merge_method() -> None: + payload = repository_ruleset_payload() + review_rule = next(rule for rule in payload["rules"] if rule["type"] == "pull_request") + review_rule["parameters"]["allowed_merge_methods"].append("rebase") + + assert audit.audit_repository_ruleset(payload) == [ + "repository ruleset must allow only merge and squash", + ] + + +def test_repository_ruleset_rejects_bypass_actors() -> None: + payload = repository_ruleset_payload() + payload["bypass_actors"] = [ + { + "actor_id": None, + "actor_type": "OrganizationAdmin", + "bypass_mode": "always", + } + ] + + assert audit.audit_repository_ruleset(payload) == [ + "repository ruleset must not configure bypass actors", + ] + + +def test_repository_ruleset_rejects_missing_bypass_evidence() -> None: + payload = repository_ruleset_payload() + del payload["bypass_actors"] + + assert audit.audit_repository_ruleset(payload) == [ + "repository ruleset must not configure bypass actors", + ] + + +def test_repository_ruleset_reports_structural_and_protection_drift() -> None: + payload = { + "id": 0, + "name": "drifted", + "source_type": "Organization", + "source": "ContextualWisdomLab", + "target": "tag", + "enforcement": "disabled", + "conditions": None, + "rules": "not-a-list", + } + + assert audit.audit_repository_ruleset(payload) == [ + "expected repository ruleset id 17921150", + "expected repository ruleset name Lock default branch", + "repository ruleset source is not ContextualWisdomLab/.github", + "repository ruleset target is not branch", + "repository ruleset enforcement is not active", + "repository ruleset must not configure bypass actors", + "repository ruleset ref scope must be exactly the default branch", + "expected one repository pull_request rule, found 0", + "repository default-branch deletion protection is missing", + "repository default-branch non-fast-forward protection is missing", + ] + + +def test_repository_ruleset_rejects_malformed_review_parameters() -> None: + payload = repository_ruleset_payload() + review_rule = next(rule for rule in payload["rules"] if rule["type"] == "pull_request") + review_rule["parameters"] = None + + assert audit.audit_repository_ruleset(payload) == [ + "repository ruleset does not require exactly two approving reviews", + "repository ruleset stale-review dismissal on push is disabled", + "repository ruleset last-push approval protection is disabled", + "repository ruleset review-thread resolution protection is disabled", + "repository ruleset must allow only merge and squash", + ] + + +def test_repository_ruleset_cli_reports_passing_policy(monkeypatch, capsys) -> None: + monkeypatch.setattr( + audit.sys, + "stdin", + StringIO(json.dumps(repository_ruleset_payload())), + ) + + assert audit.main(["--repository"]) == 0 + assert ( + "PASS: repository ruleset 17921150 protects the default branch" + in capsys.readouterr().out + ) + + +def test_ref_scope_rejects_all_branch_and_extra_proposal_branch_targets() -> None: + for include in ( + ["~ALL"], + ["~DEFAULT_BRANCH", "~ALL"], + ["~DEFAULT_BRANCH", "refs/heads/feature/*"], + ): + payload = ruleset_payload() + payload["conditions"]["ref_name"]["include"] = include + + assert audit.audit_ruleset(payload) == [ + "central ruleset ref scope must be exactly the default branch" + ] + + +def test_ref_scope_rejects_branch_exclusions() -> None: + """The strict default-branch ruleset must not hide excluded refs.""" + + payload = ruleset_payload() + payload["conditions"]["ref_name"]["exclude"] = ["refs/heads/release/*"] + + assert audit.audit_ruleset(payload) == [ + "central ruleset ref scope must be exactly the default branch" + ] + + +def test_ref_scope_rejects_string_include() -> None: + """The ruleset API contract requires an exact include list.""" + payload = ruleset_payload() + payload["conditions"]["ref_name"]["include"] = "~ALL" + + assert audit.audit_ruleset(payload) == [ + "central ruleset ref scope must be exactly the default branch" + ] + + +def test_workflows_must_not_block_branch_create_transition() -> None: + payload = ruleset_payload() + workflow_rule = next(rule for rule in payload["rules"] if rule["type"] == "workflows") + workflow_rule["parameters"]["do_not_enforce_on_create"] = False + + assert audit.audit_ruleset(payload) == [ + "central required workflows block the branch create transition" + ] + + +def test_multiple_workflow_rules_do_not_invent_create_transition_drift() -> None: + """Report structural multiplicity without attributing a missing flag to it.""" + payload = ruleset_payload() + payload["rules"].append(payload["rules"][0].copy()) + + errors = audit.audit_ruleset(payload) + + assert "expected one workflows rule, found 2" in errors + assert "central required workflows block the branch create transition" not in errors def test_expected_stacked_ruleset_passes(monkeypatch, capsys) -> None: payload = stacked_ruleset_payload() payload["rules"][0]["parameters"]["workflows"][0]["sha"] = "a" * 40 @@ -296,9 +545,10 @@ def test_audit_reports_all_structural_and_protection_drift() -> None: "expected ruleset name CWL Central required workflows", "central ruleset target is not branch", "central ruleset enforcement is not active", + "central ruleset must not configure bypass actors", "central ruleset does not include all repositories", "central ruleset repository exclusions drifted: expected ['.github', 'IRT-bibliography-set', 'noema'], got []", - "central ruleset does not target every default branch", + "central ruleset ref scope must be exactly the default branch", "expected one workflows rule, found 0", "missing central required workflow .github/workflows/close-empty-pr.yml", "missing central required workflow .github/workflows/noema-review.yml", @@ -336,7 +586,7 @@ def test_audit_reports_malformed_duplicate_workflows_and_weak_review_parameters( assert "stale-review dismissal on push is disabled" in errors assert "last-push approval protection is disabled" in errors assert "review-thread resolution protection is disabled" in errors - assert "merge and squash are not both allowed merge methods" in errors + assert "only merge and squash may be allowed merge methods" in errors def test_audit_handles_malformed_rule_parameter_shapes() -> None: @@ -379,6 +629,12 @@ def test_scheduled_audit_and_rollout_document_semgrep_and_noema_requirements() - assert "Ruleset audit could not read inherited organization ruleset" in workflow assert 'STACKED_RULESET_ID: "21732164"' in workflow assert "audit_central_required_workflows.py --stacked" in workflow + assert 'REPOSITORY_RULESET_ID: "17921150"' in workflow + assert ( + "repos/${ORG_LOGIN}/.github/rulesets/${REPOSITORY_RULESET_ID}" + in workflow + ) + assert "audit_central_required_workflows.py --repository" in workflow assert "CWL Stacked OpenCode required workflow" in rollout assert 'ref_name.exclude=["~DEFAULT_BRANCH"]' in rollout assert "- `.github/workflows/noema-review.yml`" in rollout