diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b4c6305..03e7839 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -24,3 +24,9 @@ jobs: - name: Check repository files run: python3 .github/scripts/check_repository.py + + - name: Test the settings check (offline fixtures) + run: python3 -m unittest discover -s tests -v + + - name: Validate Atrinik's baseline file + run: python3 check_settings.py --validate baselines/atrinik.json diff --git a/.gitignore b/.gitignore index 84c048a..4ea31d4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /build/ +__pycache__/ diff --git a/AGENTS.md b/AGENTS.md index 2c69bf4..ec1a6d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,36 +1,55 @@ # Atrinik GitHub settings repository guide -- This repository is the home of Atrinik's GitHub organization settings - record. Read `README.md` first. Today it holds `config/planning.json` and a - small CI check; a read-only settings check and Atrinik's baseline file are - coming in a later pull request. +- This repository holds Atrinik's settings baseline + (`baselines/atrinik.json`) and the read-only check that compares it with + the live organization (`check_settings.py`). Read `README.md` first. +- Run the check before and after any settings change, and report any + difference you did not cause first: + `python3 check_settings.py baselines/atrinik.json`. Exit 0 is a match, 1 + differences, 2 only "could not verify", 3 could not run. The organization + webhooks line is expected: the everyday login lacks `admin:org_hook`. +- How a change is made: change the setting in GitHub under the owner's login, + only when the current task authorizes that change; run the check; update + the baseline until the check is clean; open a pull request here that quotes + the clean output. Live state wins and the record catches up in the same + task. Never record a change first and apply it later. +- The check only reads. Never run its fix commands unless the current task + authorizes that exact change. Never add code to it that writes, stores a + token or runs on a schedule. +- Keep within its limits: nothing that writes, no dates that fail it, at + most three repository profiles, about ten per-repository exceptions, about + 500 lines, standard library only. If a change would cross one, simplify + instead. +- Another organization's baseline can be checked by path. Keep a private + organization's baseline, its check output and its names out of this public + repository, its pull requests and its CI logs. Test fixtures are invented. - The settings automation that used to live here (publisher, Project - synchronizer, drift audit, Project health check, community-health generator, - semantic release) was retired on 2026-11-14 by owner decision: there is no - standing organization-owner credential any more, and the rules now live in - organization rulesets that target all repositories. Do not restore, run or - imitate those tools, including from an old checkout or release. They are in - the Git history and in release `v2.26.0` for reference only. -- Settings are changed by hand under the owner's login, only when the current - task authorizes that change. Record each change in this repository in the - same task, through a pull request. Until the baseline file lands, record it - in the description of the pull request or task that made it; do not invent - a record file here. + synchronizer, drift audit, Project health check, community-health + generator, semantic release) was retired on 2026-11-14 by owner decision. + Do not restore, run or imitate it, including from an old checkout or + release. It is in the Git history and in release `v2.26.0` for reference + only. - The **Atrinik work** Project is no longer filled automatically. Keep `config/planning.json` and its Status names as they are: agent skills read them. - `atrinik/.github` is edited by hand, through pull requests in that repository. Nothing here generates it. -- Settled rollouts: required full-SHA action pinning is on; CodeQL as a merge - requirement and the `playtester` merge queue are dropped. - Keep the two required check names exactly: the `Policy validation` job in `.github/workflows/check.yml` and `Conventional PR title` in `.github/workflows/pr-title.yml`. The ruleset `03 - Required CI - github-settings` requires both. - Pin every action to a full commit SHA with a version comment. The organization requires it. -- Before you push, run `python3 .github/scripts/check_repository.py` and - `git diff --check`. Run actionlint when workflows change, if you have it. +- Before you push, run: + + ```sh + python3 .github/scripts/check_repository.py + python3 -m unittest discover -s tests + python3 check_settings.py --validate baselines/atrinik.json + git diff --check + ``` + + Run actionlint when workflows change, if you have it. - Commits and pull-request titles use Conventional Commits. - Never print or commit tokens, secrets or API responses that contain - credentials. + credentials. The baseline records names only. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a75c8f..ef4b918 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,15 +7,23 @@ Before you push, run: ```sh python3 .github/scripts/check_repository.py +python3 -m unittest discover -s tests +python3 check_settings.py --validate baselines/atrinik.json git diff --check ``` -The check confirms that every JSON file parses, that every relative link in -the Markdown files points at a file that exists, and that every action in the -workflows is pinned to a full commit SHA. Run +The first command confirms that every JSON file parses, that every relative +link in the Markdown files points at a file that exists, and that every action +in the workflows is pinned to a full commit SHA. The tests run the settings +check offline against invented fixtures in `tests/fixtures/`. Run [actionlint](https://github.com/rhysd/actionlint) as well when you change a workflow. +A pull request that changes `baselines/atrinik.json` follows a settings change +that was already made. Quote the output of +`python3 check_settings.py baselines/atrinik.json` in its description: empty +when everything matches, otherwise each remaining line and why it remains. + Keep `config/planning.json` unchanged unless the **Atrinik work** Project itself changes: agent skills read its Status names. diff --git a/README.md b/README.md index b6d71b6..d6f0654 100644 --- a/README.md +++ b/README.md @@ -1,59 +1,201 @@ # Atrinik GitHub settings -This repository is the home of Atrinik's GitHub organization settings record. -Today it holds very little: +This repository is the home of Atrinik's GitHub organization settings record: +- [`check_settings.py`](check_settings.py): a read-only check that compares + an organization's live settings with a baseline file. +- [`baselines/atrinik.json`](baselines/atrinik.json): Atrinik's baseline, + written from the live organization on 2026-11-14. - [`config/planning.json`](config/planning.json): the **Atrinik work** Project and its status names. Agent skills read the status names from here. -- A small CI check ([`.github/scripts/check_repository.py`](.github/scripts/check_repository.py)) - that keeps the files in this repository valid. +- [`tests/`](tests): offline tests with invented fixtures, and a small check + of this repository's own files + ([`.github/scripts/check_repository.py`](.github/scripts/check_repository.py)). -A read-only settings check and Atrinik's settings baseline will be added here -by a later pull request. +GitHub holds the rules. Organization rulesets aim at all repositories instead +of lists of names, so a new repository is covered without being registered +anywhere. The check and the baseline only say whether the live organization +still looks the way it was last recorded. -## The old automation is retired +## Run the check -On 2026-11-14 the owner retired the automation that used to live here: the -settings publisher, the Project synchronizer, the drift audit, the Project -health check, the community-health generator and semantic release. They were -removed because: - -- They needed a standing organization-owner credential stored in GitHub. That - credential is gone and will not be replaced. -- The rules they managed now live in organization rulesets that target all - repositories, so a new repository is covered without being registered - anywhere. -- The replacement is a read-only check and a short baseline file, both coming - to this repository. - -## How settings change now - -- The owner, or an agent session she authorizes, changes settings by hand - under her own login, in GitHub's web pages or with `gh`. -- The change is recorded here in the same task, through a pull request. Until - the baseline file lands, it is recorded in the description of the pull - request or task that made it. -- The **Atrinik work** Project board is no longer filled automatically. Add - issues and pull requests to it by hand when they belong there. -- [`atrinik/.github`](https://github.com/atrinik/.github) (issue forms, - pull-request template, contribution guidance, public profile) is no longer - generated from here. Edit it by hand through pull requests in that - repository. +You need Python 3.10 or newer and `gh`, logged in as someone who can read the +organization's settings (for Atrinik, the owner). + +```sh +python3 check_settings.py baselines/atrinik.json +``` + +It prints nothing and exits 0 when everything matches. Otherwise it prints one +line per difference, each ending with `| fix:` and what would make the live +organization match the baseline, usually a command, then a blank line and the +things it could not verify: + +```text +atrinik/pathfinder: allow_merge_commit is true, baseline false | fix: gh api --method PATCH repos/atrinik/pathfinder -F allow_merge_commit=false +could not verify: orgs/atrinik webhooks: Not Found (HTTP 404) (the login lacks the admin:org_hook scope) +``` + +| Exit | Meaning | +| --- | --- | +| 0 | Everything matches. | +| 1 | At least one difference. | +| 2 | No difference, but something could not be verified. | +| 3 | The check could not run: bad arguments, an invalid baseline, no `gh`, `gh` not logged in or its token rejected, an unreadable repository list, or any unexpected error, such as an API response or a baseline shape it does not know (the traceback goes to standard error). Exit 1 always means differences, never a crash. The run stopped and printed nothing on standard output, so any difference it had found is not shown: fix the cause and rerun it. A single read that fails or times out (for example one network blip) is a "could not verify" line instead, and the run goes on. | + +The check never runs a fix itself. A fix command is a suggestion: when live +state is right and the record is out of date, update the baseline instead. + +Other uses: + +```sh +python3 check_settings.py --validate baselines/atrinik.json # the file only (keys, shapes, limits), offline +python3 check_settings.py baselines/atrinik.json --ruleset '02 - Changes through pull requests' +``` + +`--validate` catches the usual mistakes in a baseline file. A shape it does +not know still ends in exit 3 at the live run. `--ruleset` prints a ruleset's +body from the baseline. The fix commands for rulesets pipe it into +`gh api --method PUT|POST ... --input -`. + +### Another organization + +The script holds no organization's data. Pass any organization's baseline by +path, for example one kept in a private repository: + +```sh +python3 check_settings.py /path/to/private/baseline.json +``` + +Keep a private organization's baseline and its check output out of this +public repository, its pull requests and its CI logs. + +### Without `admin:org` + +If the login lacks `admin:org`, the per-repository checks still run: merge +settings, effective rules on each default branch and the tag rules, security +configuration attachment, team grants, environments and repository webhooks. +Organization-level reads (Actions policy, organization rulesets, security +configurations, teams, installed Apps) and settings GitHub hides from that +login become "could not verify" lines. When the login does see the whole +response (an owner with `admin:org` for the organization, admin on the +repository for merge settings, and always for Actions policy and security +configurations), a baseline field the response lacks is a difference +("is not in the API response"): usually a misspelled field name. Organization webhooks need +`admin:org_hook`, which the everyday login does not have, so that line is +expected. -## Rollouts that were never switched on +## What it checks -The old configuration declared three rollouts that were never switched on. -They are now settled: +**Policy**, compared field by field: -- Required full-SHA pinning of actions is on for the organization. -- CodeQL as a merge requirement is dropped. CodeQL alerts stay advisory. -- The merge queue pilot on `playtester` is dropped. +- organization settings (`organization_settings`); +- Actions policy, allowed actions and default workflow permissions + (`actions`); +- organization rulesets (`rulesets`, plus one required-check ruleset per + repository generated from `required_check_ruleset`); a ruleset missing + from either side is a difference; +- each repository's effective rules on its default branch, its required + checks, and the rules on its release tags (`branch_rules`, `checks`, + `tag_rules`). Branch rules come from GitHub's own view of what applies to + the default branch, so they also catch a repository-level ruleset or an + organization ruleset that stopped targeting the repository. Every rule of + a type must match the profile's parameters, and a rule type the profile + does not name is a difference. So list every parameter of a rule in the + profile, as `baselines/atrinik.json` does for `pull_request`, or a + repository ruleset could tighten or loosen the unlisted ones unseen. + These lines name the ruleset a rule comes from and always print, even + when that ruleset has a line of its own. They carry a command only for a + repository ruleset (`repos/O/R/rulesets/ID`). For an organization ruleset + the command is on that ruleset's own line, which exists when the ruleset + is missing, unrecorded or different from the baseline (a "could not + verify" line if it could not be read). When there is neither, the + baseline's rulesets and the profile disagree, and the fix is in the + baseline. Tag rules are checked for presence only, and a tag + ruleset with ref excludes does not count; if a tag ruleset's body cannot + be read, the tag check is one "could not verify" line; +- security configurations, which one is the default for new repositories, + and which one each repository has attached (`security_configurations`, + `security_configuration`); +- each repository's merge settings (`repository`). -## History +**Changed since last recorded**, names only (`recorded`): teams and their +repository grants, installed Apps and their repository selection mode +(`all` or `selected`), environment names, and webhook hosts. A line here +means something changed since the baseline was written, not that it is +wrong. -The retired tooling, its configuration and its documentation are in the Git -history. The last release that contains them is -[`v2.26.0`](https://github.com/atrinik/github-settings/releases/tag/v2.26.0). +**Repositories it cannot classify fail it.** Every repository in the +organization must be listed under `repositories` with a profile, and every +listed one must exist. + +## The baseline file + +One JSON file per organization: + +| Key | What it holds | +| --- | --- | +| `organization` | The organization's login. | +| `notes` | Free text for people. Not checked. | +| `organization_settings`, `actions` | Values compared with the live ones. | +| `rulesets` | Organization rulesets by name; each is the body a `PUT` would send. | +| `required_check_ruleset`, `check_integration_id` | The template for the one-per-repository required-check ruleset (`{repository}` is replaced) and the App that must report the checks (15368 is GitHub Actions). | +| `security_configurations` | Configurations by name, with `default_for_new_repos`. | +| `profiles` | At most three repository profiles: `repository` (merge settings), `branch_rules`, `tag_rules`, `security_configuration`. | +| `repositories` | Every repository: its `profile`, its required `checks`, and rarely `overrides` of profile keys with a `why`. | +| `recorded` | Names only: `teams`, `apps`, `environments`, `webhooks`, `organization_webhooks`. | +| `reminders` | Plain notes about credentials and Apps, such as rotation dates. Never checked. | + +Values are compared as subsets: a field GitHub adds later does not make a +difference, and lists match in any order. + +## How a change is made + +Live state wins and the record catches up in the same task: + +1. Run the check. Report any difference you did not cause first, before you + change anything. +2. Make the change in GitHub, under the owner's login, only when the current + task authorizes it. +3. Run the check again. It shows exactly what you changed. +4. Update the baseline until the check is clean, and open a pull request + here that quotes the clean output (or explains each remaining line). + +Changes made by hand in GitHub's web pages show up at the next run. GitHub +keeps its own history too: the organization audit log and each ruleset's +history. + +## Limits + +These keep the check small enough to read in one sitting. Simplify instead of +growing past them. + +- Nothing that writes. The script only runs `gh api -X GET`; the tests check + that. +- No dates that fail it. Rotation and review dates are reminders only. +- At most three repository profiles and about ten per-repository + exceptions (`overrides`). `--validate` enforces three and ten. +- About 500 lines for the script. A test stops it at 550. +- Standard library only, and no stored token: it runs as whoever runs it, + by hand. Nothing runs it on a schedule. + +## The old automation is retired + +On 2026-11-14 the owner retired the automation that used to live here: the +settings publisher, the Project synchronizer, the drift audit, the Project +health check, the community-health generator and semantic release. It needed +a standing organization-owner credential stored in GitHub; that credential is +gone and will not be replaced. The tooling is in the Git history and in +release +[`v2.26.0`](https://github.com/atrinik/github-settings/releases/tag/v2.26.0) +for reference only. + +- The **Atrinik work** Project board is no longer filled automatically. Add + issues and pull requests to it by hand when they belong there. +- [`atrinik/.github`](https://github.com/atrinik/.github) is edited by hand + through pull requests in that repository. +- Required full-SHA action pinning is on. CodeQL is not a merge requirement; + its alerts stay advisory. The `playtester` merge queue pilot is dropped. ## Contributing diff --git a/baselines/atrinik.json b/baselines/atrinik.json new file mode 100644 index 0000000..707c7fd --- /dev/null +++ b/baselines/atrinik.json @@ -0,0 +1,603 @@ +{ + "organization": "atrinik", + "notes": [ + "Atrinik's settings baseline, read by check_settings.py. Written from the live organization on 2026-11-14, after the changes the owner approved that day.", + "Live state wins: change the setting first, run the check, then update this file in a pull request that quotes the clean output.", + "\"recorded\" holds names only. A difference there means something changed since it was last recorded, not that it is wrong.", + "\"reminders\" are plain notes for the owner. The check never reads them and never fails on a date." + ], + "organization_settings": { + "default_repository_permission": "none", + "members_can_create_repositories": false, + "members_can_create_public_repositories": false, + "members_can_create_private_repositories": false, + "members_can_create_internal_repositories": false, + "members_can_fork_private_repositories": false, + "members_can_delete_repositories": false, + "members_can_change_repo_visibility": false, + "members_can_invite_outside_collaborators": true, + "members_can_delete_issues": false, + "members_can_create_teams": false, + "members_can_create_pages": true, + "web_commit_signoff_required": false, + "deploy_keys_enabled_for_repositories": false, + "two_factor_requirement_enabled": true, + "default_repository_branch": "main" + }, + "actions": { + "permissions": { + "enabled_repositories": "all", + "allowed_actions": "selected", + "sha_pinning_required": true + }, + "selected_actions": { + "github_owned_allowed": true, + "patterns_allowed": [ + "atrinik/*", + "docker/login-action@*", + "docker/setup-buildx-action@*", + "docker/build-push-action@*", + "codecov/codecov-action@*", + "rust-lang/crates-io-auth-action@*" + ], + "verified_allowed": false + }, + "workflow": { + "default_workflow_permissions": "read", + "can_approve_pull_request_reviews": false + } + }, + "rulesets": { + "01 - Default branch integrity": { + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + } + ], + "bypass_actors": [] + }, + "01 - Default branch linear history": { + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "rules": [ + { + "type": "required_linear_history" + } + ], + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ] + }, + "02 - Changes through pull requests": { + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "rules": [ + { + "type": "pull_request", + "parameters": { + "required_approving_review_count": 0, + "dismiss_stale_reviews_on_push": false, + "required_reviewers": [], + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_review_thread_resolution": true, + "require_extra_approval_for_unattributed_changes": true, + "allowed_merge_methods": [ + "squash" + ] + } + } + ], + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ] + }, + "04 - Immutable release tags": { + "target": "tag", + "enforcement": "active", + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/tags/v[0-9]*" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "update" + } + ], + "bypass_actors": [] + } + }, + "required_check_ruleset": { + "name": "03 - Required CI - {repository}", + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "{repository}" + ], + "protected": false + } + }, + "rules": [ + { + "type": "required_status_checks", + "parameters": { + "strict_required_status_checks_policy": true, + "do_not_enforce_on_create": false + } + } + ], + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ] + }, + "check_integration_id": 15368, + "security_configurations": { + "Atrinik security baseline": { + "description": "Required security features for all Atrinik repositories.", + "advanced_security": "enabled", + "dependency_graph": "enabled", + "dependency_graph_autosubmit_action": "disabled", + "dependency_graph_autosubmit_action_options": { + "labeled_runners": false + }, + "dependabot_alerts": "enabled", + "dependabot_security_updates": "enabled", + "dependabot_delegated_alert_dismissal": "disabled", + "code_scanning_options": { + "allow_advanced": false + }, + "code_scanning_default_setup": "enabled", + "code_scanning_default_setup_options": { + "runner_type": "not_set", + "runner_label": null + }, + "code_scanning_delegated_alert_dismissal": "disabled", + "secret_scanning": "enabled", + "secret_scanning_push_protection": "enabled", + "secret_scanning_delegated_bypass": "disabled", + "secret_scanning_non_provider_patterns": "disabled", + "secret_scanning_delegated_alert_dismissal": "disabled", + "enforcement": "enforced", + "private_vulnerability_reporting": "enabled", + "secret_scanning_validity_checks": "enabled", + "secret_scanning_extended_metadata": "disabled", + "secret_scanning_generic_secrets": "disabled", + "secret_scanning_generic_secrets_push_protection": "disabled", + "default_for_new_repos": "all" + }, + "Atrinik advanced CodeQL baseline": { + "description": "Atrinik security baseline for repositories with governed CodeQL advanced setup.", + "advanced_security": "enabled", + "dependency_graph": "enabled", + "dependency_graph_autosubmit_action": "disabled", + "dependency_graph_autosubmit_action_options": { + "labeled_runners": false + }, + "dependabot_alerts": "enabled", + "dependabot_security_updates": "enabled", + "dependabot_delegated_alert_dismissal": "disabled", + "code_scanning_options": { + "allow_advanced": true + }, + "code_scanning_default_setup": "enabled", + "code_scanning_default_setup_options": { + "runner_type": "not_set", + "runner_label": null + }, + "code_scanning_delegated_alert_dismissal": "disabled", + "secret_scanning": "enabled", + "secret_scanning_push_protection": "enabled", + "secret_scanning_delegated_bypass": "disabled", + "secret_scanning_non_provider_patterns": "disabled", + "secret_scanning_delegated_alert_dismissal": "disabled", + "enforcement": "enforced", + "private_vulnerability_reporting": "enabled", + "secret_scanning_validity_checks": "enabled", + "secret_scanning_extended_metadata": "disabled", + "secret_scanning_generic_secrets": "disabled", + "secret_scanning_generic_secrets_push_protection": "disabled" + } + }, + "profiles": { + "standard": { + "repository": { + "allow_squash_merge": true, + "allow_merge_commit": false, + "allow_rebase_merge": false, + "delete_branch_on_merge": true, + "squash_merge_commit_title": "PR_TITLE", + "squash_merge_commit_message": "PR_BODY", + "archived": false + }, + "branch_rules": { + "deletion": {}, + "non_fast_forward": {}, + "required_linear_history": {}, + "pull_request": { + "required_approving_review_count": 0, + "dismiss_stale_reviews_on_push": false, + "required_reviewers": [], + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_review_thread_resolution": true, + "require_extra_approval_for_unattributed_changes": true, + "allowed_merge_methods": [ + "squash" + ] + } + }, + "tag_rules": { + "ref": "refs/tags/v[0-9]*", + "rules": [ + "deletion", + "non_fast_forward", + "update" + ] + }, + "security_configuration": "Atrinik security baseline" + }, + "archived": { + "repository": { + "archived": true + }, + "security_configuration": "Atrinik security baseline" + } + }, + "repositories": { + ".github": { + "profile": "standard", + "checks": [] + }, + "agent-integrations": { + "profile": "standard", + "checks": [ + "Required checks", + "Conventional PR title" + ] + }, + "atrinik": { + "profile": "standard", + "checks": [ + "Integration validation", + "Conventional PR title" + ] + }, + "classic": { + "profile": "standard", + "checks": [ + "Classic validation", + "CodeQL validation", + "Conventional PR title" + ], + "overrides": { + "security_configuration": "Atrinik advanced CodeQL baseline" + }, + "why": "Classic runs CodeQL itself (advanced setup in .github/workflows/codeql.yml, whose CodeQL validation job is a required check), which the default-setup configuration does not allow." + }, + "client": { + "profile": "standard", + "checks": [ + "Client validation", + "Conventional PR title" + ] + }, + "content": { + "profile": "standard", + "checks": [ + "Content validation", + "Conventional PR title" + ] + }, + "content-toolkit": { + "profile": "standard", + "checks": [ + "Content toolkit validation", + "Conventional PR title" + ] + }, + "deploy-control": { + "profile": "standard", + "checks": [ + "Deploy-control validation", + "Conventional PR title" + ] + }, + "devcontainer": { + "profile": "standard", + "checks": [ + "Required checks", + "Conventional PR title" + ] + }, + "github-settings": { + "profile": "standard", + "checks": [ + "Policy validation", + "Conventional PR title" + ] + }, + "metaserver-worker": { + "profile": "standard", + "checks": [ + "verify", + "Conventional PR title" + ] + }, + "nawerhals": { + "profile": "archived" + }, + "observatory": { + "profile": "standard", + "checks": [ + "Validate dashboard and Workers", + "Conventional PR title" + ] + }, + "pathfinder": { + "profile": "standard", + "checks": [ + "Pathfinder validation", + "Conventional PR title" + ] + }, + "playtester": { + "profile": "standard", + "checks": [ + "Playtester validation", + "Conventional PR title" + ] + }, + "protocol": { + "profile": "standard", + "checks": [ + "Protocol validation", + "Conventional PR title" + ] + }, + "resources": { + "profile": "standard", + "checks": [ + "Asset validation", + "Conventional PR title" + ] + }, + "server": { + "profile": "standard", + "checks": [ + "Server validation", + "Conventional PR title" + ] + }, + "service-updater": { + "profile": "standard", + "checks": [ + "Updater validation", + "Conventional PR title" + ] + }, + "sound": { + "profile": "standard", + "checks": [ + "Asset validation", + "Conventional PR title" + ] + }, + "tools": { + "profile": "standard", + "checks": [ + "Tools validation", + "Conventional PR title" + ] + }, + "web-platform": { + "profile": "standard", + "checks": [] + }, + "website": { + "profile": "standard", + "checks": [ + "Website validation", + "Conventional PR title" + ] + } + }, + "recorded": { + "teams": { + "admins": { + "atrinik": "admin", + "nawerhals": "admin", + "observatory": "admin", + "sound": "admin" + }, + "developers": {}, + "maintainers": { + ".github": "maintain", + "agent-integrations": "maintain", + "atrinik": "maintain", + "classic": "maintain", + "client": "maintain", + "content": "maintain", + "content-toolkit": "maintain", + "deploy-control": "maintain", + "devcontainer": "maintain", + "github-settings": "maintain", + "metaserver-worker": "maintain", + "observatory": "maintain", + "pathfinder": "maintain", + "playtester": "maintain", + "protocol": "maintain", + "resources": "maintain", + "server": "maintain", + "service-updater": "maintain", + "sound": "maintain", + "tools": "maintain", + "web-platform": "maintain", + "website": "maintain" + }, + "testers": {} + }, + "apps": { + "atrinik-classic-dependency-updater": "selected", + "atrinik-crate-dependency-update": "selected", + "claude": "all", + "cloudflare-workers-and-pages": "selected", + "codecov": "selected" + }, + "environments": { + "classic": [ + "discord-release", + "github-pages" + ], + "content-toolkit": [ + "crates-io-release" + ], + "metaserver-worker": [ + "development", + "production" + ], + "observatory": [ + "production" + ], + "protocol": [ + "crates-io-bootstrap", + "crates-io-release" + ] + }, + "webhooks": { + "classic": [ + "atrinik.org" + ] + }, + "organization_webhooks": [] + }, + "reminders": [ + { + "item": "App atrinik-classic-dependency-updater (classic: variable DEPENDENCY_UPDATE_APP_ID, secret DEPENDENCY_UPDATE_APP_PRIVATE_KEY)", + "note": "Key rotation was due 2026-11-09. The key still works. Rotate it next time you are at Lyra, or decide it can wait." + }, + { + "item": "App atrinik-crate-dependency-update (client: variable CRATE_DEPENDENCY_UPDATE_APP_ID, secret CRATE_DEPENDENCY_UPDATE_APP_PRIVATE_KEY)", + "note": "Key created 2026-10-11. Rotate it around 2027-01-09." + }, + { + "item": "metaserver-worker environments development and production: secrets CLOUDFLARE_API_TOKEN and ATRINIK_DELIVERY_RESOURCE_READ_TOKEN", + "note": "Cloudflare API tokens, separate for each environment. Rotate them around 2027-01-09." + }, + { + "item": "observatory environment production: secrets CLOUDFLARE_DEPLOY_API_TOKEN and CLOUDFLARE_D1_API_TOKEN", + "note": "Cloudflare API tokens. No rotation date is recorded; note one here when you next rotate them. Deployments wait for approval by zoeyrose." + }, + { + "item": "App cloudflare-workers-and-pages (selected repositories)", + "note": "Last recorded on website and metaserver-worker; its review was due 2026-11-13. The gh login cannot read which repositories an App reaches; look once in the organization's GitHub Apps settings." + }, + { + "item": "App codecov (selected repositories)", + "note": "Last recorded on atrinik, classic, client, content-toolkit, protocol and server. Confirm in the organization's GitHub Apps settings." + }, + { + "item": "App claude (all repositories)", + "note": "The Claude GitHub App reaches every repository, including new ones." + }, + { + "item": "crates.io trusted publishing", + "note": "protocol publishes atrinik-protocol and content-toolkit publishes atrinik-artifact, atrinik-catalog, atrinik-diagnostics, atrinik-schema and atrinik-source, each from its crates-io-release environment. On crates.io, remove atrinik/renderer as a trusted publisher: that repository is no longer in the organization." + }, + { + "item": "Retired credential ATRINIK_SETTINGS_TOKEN", + "note": "Deleted from github-settings on 2026-11-14. Delete its entry at github.com/settings/tokens too; deleting the secret does not revoke the token." + }, + { + "item": "Organization webhooks", + "note": "The gh login cannot read them (needs admin:org_hook), so the check reports them as could not verify. Recorded as none; look once at the organization's Webhooks page and correct the list if needed." + } + ] +} diff --git a/check_settings.py b/check_settings.py new file mode 100644 index 0000000..2cb846b --- /dev/null +++ b/check_settings.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +"""Read-only check of one GitHub organization against its baseline file. + +Usage: check_settings.py BASELINE.json [--validate | --ruleset NAME]. It runs only `gh api -X GET` as whoever +is logged in and writes nothing. Exit: 0 match, 1 differences, 2 only "could not verify", 3 could not run. +README.md has the output, the baseline format and the limits. +""" + +import argparse +import concurrent.futures +import copy +import functools +import json +import re +import shlex +import subprocess +import sys +import traceback +import urllib.parse + +MAX_PROFILES = 3 +MAX_EXCEPTIONS = 10 +TOP_KEYS = {"organization", "organization_settings", "actions", "rulesets", "required_check_ruleset", "reminders", + "check_integration_id", "security_configurations", "profiles", "repositories", "recorded", "notes"} +ACTIONS_PATHS = {"permissions": "", "selected_actions": "/selected-actions", "workflow": "/workflow"} + + +def mapping(of=lambda v: True): + """A shape test: an object whose values all pass `of`.""" + return lambda v: isinstance(v, dict) and all(of(x) for x in v.values()) + + +def names_list(v): + return isinstance(v, list) and all(isinstance(x, str) for x in v) + + +SHAPES = {"repository": ("an object of fields", mapping()), # profile keys, as the check reads them + "branch_rules": ("rule type to parameters", mapping(mapping())), + "tag_rules": ('a "ref" and a list of rule types in "rules"', + lambda v: isinstance(v, dict) and isinstance(v.get("ref"), str) and names_list(v.get("rules"))), + "security_configuration": ("a name", lambda v: isinstance(v, str))} +RECORDED = {"teams": mapping(mapping(lambda v: isinstance(v, str))), "apps": mapping(lambda v: isinstance(v, str)), + "environments": mapping(names_list), "webhooks": mapping(names_list), "organization_webhooks": names_list} +RULESET = lambda v: (isinstance(v, dict) and {"target", "enforcement"} <= set(v) and isinstance( # noqa: E731 + v.get("rules"), list) and all(isinstance(r, dict) and isinstance(r.get("type"), str) + and isinstance(r.get("parameters", {}), dict) for r in v["rules"])) +SECTIONS = {"organization": ("the organization's login", lambda v: isinstance(v, str) and v != ""), + "organization_settings": ("an object of fields", mapping()), # top-level sections the check reads + "rulesets": ("name to target, enforcement and rules with a type and object parameters", mapping(RULESET)), + "required_check_ruleset": ("a rulesets entry with {repository} in its name", + lambda v: RULESET(v) and "{repository}" in str(v.get("name"))), + "reminders": ('a list of objects with an "item" and a "note"', lambda v: isinstance(v, list) and all( + isinstance(i, dict) and isinstance(i.get("item"), str) and isinstance(i.get("note"), str) for i in v)), + "actions": ("permissions, selected_actions or workflow, each an object of fields", + lambda v: mapping(mapping())(v) and set(v) <= set(ACTIONS_PATHS)), + "security_configurations": ("configuration name to an object of fields", mapping(mapping())), + "repositories": ("repository name to an object with a profile", mapping(lambda v: isinstance(v, dict))), + "recorded": (f"{', '.join(RECORDED)}, each in its shape (README.md)", + lambda v: isinstance(v, dict) and all(k in RECORDED and RECORDED[k](x) for k, x in v.items()))} +# Organization settings the REST API cannot change: the fix is this settings page. +WEB_ONLY = dict.fromkeys(["members_can_delete_repositories", "members_can_change_repo_visibility", + "members_can_delete_issues", "members_can_invite_outside_collaborators", + "members_can_create_teams", "readers_can_create_discussions"], "member_privileges") +WEB_ONLY.update(two_factor_requirement_enabled="security", default_repository_branch="repository_defaults") +show = functools.partial(json.dumps, sort_keys=True) +# Effective rules are GitHub's computed view of a branch or tag, so their lines name the ruleset and carry no command. +SEE_RULESETS = ("an organization ruleset that is missing, unrecorded or different from the baseline has its own " + "line with the command (a could-not-verify line if it could not be read); otherwise correct the " + "ruleset by hand, or the baseline") + + +class Unverifiable(Exception): + """A read that failed: missing scope or missing access.""" + + +class CannotRun(Exception): + """gh is not logged in, or the repository list could not be read.""" + + +def parse_output(text): + """gh prints one JSON value per page; join pages of a list into one list.""" + decoder, text, pos, values = json.JSONDecoder(), text.strip(), 0, [] + while pos < len(text): + value, pos = decoder.raw_decode(text, pos) + values.append(value) + pos = len(text) - len(text[pos:].lstrip()) + if len(values) < 2: + return values[0] if values else None + return [item for value in values for item in value] + + +def gh_fetch(path, paginate): + command = ["gh", "api", "-X", "GET", path] + (["--paginate"] if paginate else []) + proc = subprocess.run(command, capture_output=True, text=True, check=False) + if proc.returncode == 0: + return parse_output(proc.stdout) + message = (proc.stderr.strip().splitlines() or ["gh api failed"])[0].removeprefix("gh: ")[:160] + scope = re.search(r'needs the "([^"]+)" scope', proc.stderr) + if not scope and re.search(r"gh auth login|HTTP 401|Bad credentials", proc.stderr, re.IGNORECASE): + raise CannotRun(message) # no read can succeed; a timeout on one read is only "could not verify" + if scope: + message += f" (the login lacks the {scope.group(1)} scope)" + elif path.startswith("orgs/") and re.search(r"HTTP 40[34]", message): + message += " (the login may lack admin:org)" + raise Unverifiable(message) + + +class GitHub: + """Cached GET reads. `get` returns parsed JSON, or the Unverifiable a failed `fetch` raised.""" + + def __init__(self, fetch=gh_fetch): + self.fetch, self.cache = fetch, {} + + def get(self, path, paginate=False): + if path not in self.cache: + try: # an empty response (204, such as no configuration attached) reads as {} + value = self.fetch(path, paginate) + self.cache[path] = {} if value is None else value + except Unverifiable as error: + self.cache[path] = error + return self.cache[path] + + def prefetch(self, reads): + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(lambda read: self.get(*read), reads)) + + +def gh_command(method, path, body=None): + command = f"gh api --method {method} {shlex.quote(path)}" + if body is None: + return command + if all(isinstance(v, (str, bool, int)) or v is None for v in body.values()): + return " ".join([command] + [("-f " if isinstance(v, str) else "-F ") + shlex.quote( + f"{k}={v if isinstance(v, str) else json.dumps(v)}") for k, v in body.items()]) + return f"echo {shlex.quote(json.dumps(body, separators=(',', ':')))} | {command} --input -" + + +def matches(want, have): + """True when `have` holds everything in `want`. Lists match in any order.""" + if isinstance(want, dict): + return isinstance(have, dict) and all(k in have and matches(v, have[k]) for k, v in want.items()) + if isinstance(want, list): + if not isinstance(have, list) or len(want) != len(have): + return False + left = list(have) + for item in want: + found = next((i for i, other in enumerate(left) if matches(item, other)), None) + if found is None: + return False + left.pop(found) + return True + return type(want) is type(have) and want == have + + +def expected_rulesets(base): + rulesets = {name: dict(body, name=name) for name, body in base.get("rulesets", {}).items()} + template = base.get("required_check_ruleset") + if template: + for repo, entry in sorted(base.get("repositories", {}).items()): + if not entry.get("checks"): + continue + body = json.loads(json.dumps(template).replace("{repository}", repo)) + app = {"integration_id": base["check_integration_id"]} if base.get("check_integration_id") else {} + for rule in (r for r in body["rules"] if r["type"] == "required_status_checks"): + rule.setdefault("parameters", {})["required_status_checks"] = [dict(app, context=c) + for c in entry["checks"]] + rulesets[body["name"]] = body + return rulesets + + +def profile_for(base, entry): + profile = copy.deepcopy(base["profiles"][entry["profile"]]) + profile.update(copy.deepcopy(entry.get("overrides", {}))) + return profile + + +def shape_errors(where, value, shapes): + """The shapes the checker reads, so a malformed baseline fails --validate instead of the live run.""" + if not isinstance(value, dict): + return [f"{where}: must be an object"] + return [f"{where}: unknown key {k!r}" for k in sorted(set(value) - set(shapes))] + [ + f"{where}: {k} must be {shapes[k][0]}" for k in sorted(set(value) & set(shapes)) if not shapes[k][1](value[k])] + + +def validate(base): + if not isinstance(base, dict): + return ["the baseline must be a JSON object"] + errors = [f"unknown top-level key {k!r}" for k in sorted(set(base) - TOP_KEYS)] + errors += shape_errors("baseline", {k: v for k, v in dict({"organization": None}, **base).items() if k in SECTIONS}, + SECTIONS) + profiles = base.get("profiles") + if not isinstance(profiles, dict) or not 1 <= len(profiles) <= MAX_PROFILES: + errors.append(f'"profiles" must hold 1 to {MAX_PROFILES} profiles') + profiles = {} + bad_profiles = set() + for name, profile in profiles.items(): + bad = shape_errors(f"profile {name}", profile, SHAPES) + errors, bad_profiles = errors + bad, bad_profiles | ({name} if bad else set()) + configs, exceptions = base.get("security_configurations", {}), 0 + repos = base.get("repositories") # a section of the wrong shape is reported above + for repo, entry in sorted(repos.items() if isinstance(repos, dict) else []): + if not isinstance(entry, dict) or entry.get("profile") not in list(profiles): # list: value may be unhashable + errors.append(f"repository {repo}: needs a known profile") + continue + if not names_list(entry.get("checks", [])): + errors.append(f"repository {repo}: checks must be a list of check names") + overrides = entry.get("overrides", {}) + bad = shape_errors(f"repository {repo} override", overrides, SHAPES) + errors, exceptions = errors + bad, exceptions + (len(overrides) if isinstance(overrides, dict) else 0) + if bad or entry["profile"] in bad_profiles: + continue + if overrides and not entry.get("why"): + errors.append(f'repository {repo}: an override needs a "why"') + wanted = profile_for(base, entry).get("security_configuration") + if wanted and isinstance(configs, dict) and wanted not in configs: # a bad section is reported above + errors.append(f"repository {repo}: security configuration {wanted!r} is not in the baseline") + if exceptions > MAX_EXCEPTIONS: + errors.append(f"{exceptions} per-repository exceptions; the limit is {MAX_EXCEPTIONS}") + return errors + + +class Checker: + def __init__(self, base, github, script="check_settings.py", baseline="BASELINE"): + self.base, self.gh, self.org = base, github, base["organization"] + self.diffs, self.unverified = [], [] + self.ruleset_cmd = f"python3 {shlex.quote(script)} {shlex.quote(baseline)} --ruleset" + self.wanted_rulesets = expected_rulesets(base) + self.live_ruleset_ids, self.config_ids, self.recorded = None, {}, base.get("recorded", {}) + + def diff(self, where, text, fix): + self.diffs.append(f"{where}: {text} | fix: {fix}") + + def changed(self, where, text, undo): + self.diffs.append(f"{where}: {text} (changed since last recorded) | fix: record it in the baseline, " + f"or undo: {undo}") + + def cannot(self, where, reason): + self.unverified.append(f"could not verify: {where}: {reason}") + + def read(self, where, path, paginate=False): + value = self.gh.get(path, paginate) + if isinstance(value, Unverifiable): + return self.cannot(where, str(value)) + return value + + def names(self, where, label, have, recorded, undo_new, undo_gone): + for name in sorted(set(have) - set(recorded)): + self.changed(where, f"{label} {name!r} is new", undo_new(name)) + for name in sorted(set(recorded) - set(have)): + self.changed(where, f"{label} {name!r} is gone", undo_gone(name)) + + def ruleset_fix(self, name): + """POST when the live list lacks it, PUT when known; ID stands in when the list could not be read.""" + known = self.live_ruleset_ids is None or name in self.live_ruleset_ids + path = f"orgs/{self.org}/rulesets" + (f"/{(self.live_ruleset_ids or {}).get(name, 'ID')}" if known else "") + return f"{self.ruleset_cmd} {shlex.quote(name)} | gh api --method {'PUT' if known else 'POST'} {path} --input -" + + def compare_fields(self, where, want, have, fix_for, needs=None): + """A field the response lacks is unverifiable when the login `needs` more access, else a baseline error.""" + hidden = ", ".join(sorted(set(want) - set(have))) + if hidden and not needs: + self.diff(where, f"{hidden} is not in the API response", "correct the field name in the baseline") + elif hidden: + self.cannot(where, f"{hidden} not visible to this login (needs {needs}, or a field name is misspelled)") + for key, value in want.items(): + if key in have and not matches(value, have[key]): + self.diff(where, f"{key} is {show(have[key])}, baseline {show(value)}", fix_for(key, value)) + + def run(self): + org = self.org # organization reads run one by one; the per-repository ones run eight at a time + self.check_organization() + self.check_actions() + self.check_rulesets() + self.check_security_configurations() + repos = self.gh.get(f"orgs/{org}/repos?per_page=100&type=all", True) + if isinstance(repos, Unverifiable): + raise CannotRun(f"cannot list the repositories, so none was checked: {repos}") + self.check_repositories({r["name"]: r for r in repos}) + self.check_apps() + + def check_organization(self): + live, page = self.read(f"orgs/{self.org}", f"orgs/{self.org}"), f"https://github.com/organizations/{self.org}" + if live is not None: + fix = lambda k, v: (f"change it at {page}/settings/{WEB_ONLY[k]}" if k in WEB_ONLY # noqa: E731 + else gh_command("PATCH", f"orgs/{self.org}", {k: v})) + # Owners with admin:org see the member privileges; without them, hidden fields are unverifiable. + self.compare_fields(f"orgs/{self.org}", self.base.get("organization_settings", {}), live, fix, + None if "members_can_create_repositories" in live else "admin:org") + + def check_actions(self): + for section, want in self.base.get("actions", {}).items(): + live = self.read(path := f"orgs/{self.org}/actions/permissions{ACTIONS_PATHS[section]}", path) + if live is not None: + self.compare_fields(path, want, live, lambda key, value, p=path, w=want: gh_command("PUT", p, w)) + + def check_rulesets(self): + listing = self.read("organization rulesets", f"orgs/{self.org}/rulesets?per_page=100", True) + if listing is None: + return + self.live_ruleset_ids = {r["name"]: r["id"] for r in listing} + self.gh.prefetch([(f"orgs/{self.org}/rulesets/{r['id']}", False) for r in listing]) + for name, want in sorted(self.wanted_rulesets.items()): + if name not in self.live_ruleset_ids: + self.diff(f"orgs/{self.org}/rulesets", f"ruleset {name!r} is missing", self.ruleset_fix(name)) + continue + live = self.read(path := f"orgs/{self.org}/rulesets/{self.live_ruleset_ids[name]}", path) + if live is not None and not matches(want, live): + keys = ", ".join(k for k in sorted(want) if not matches(want[k], live.get(k))) + self.diff(path, f"ruleset {name!r} differs in {keys}", self.ruleset_fix(name)) + for name, rid in sorted(self.live_ruleset_ids.items()): + if name not in self.wanted_rulesets: + self.diff(f"orgs/{self.org}/rulesets/{rid}", f"ruleset {name!r} is not in the baseline", + f"record it, or {gh_command('DELETE', f'orgs/{self.org}/rulesets/{rid}')}") + + def check_security_configurations(self): + path = f"orgs/{self.org}/code-security/configurations" + live = self.read("security configurations", f"{path}?per_page=100", True) + defaults = self.read("security configuration defaults", f"{path}/defaults") + if live is None: + return + live = {c["name"]: c for c in live if c.get("target_type") != "global"} + self.config_ids = {name: c["id"] for name, c in live.items()} + default_of = {d["configuration"]["id"]: d["default_for_new_repos"] for d in defaults or []} + for name, want in sorted(self.base.get("security_configurations", {}).items()): + want = dict({"default_for_new_repos": "none"}, **want) # not a default unless the baseline says so + if name not in live: + body = {k: v for k, v in want.items() if k != "default_for_new_repos"} + self.diff(path, f"security configuration {name!r} is missing", gh_command("POST", path, body)) + continue + target, have = f"{path}/{live[name]['id']}", dict(live[name], default_for_new_repos=default_of.get( + live[name]["id"], "none")) + if defaults is None: # already a could-not-verify line + del want["default_for_new_repos"] + self.compare_fields(f"{target} ({name})", want, have, lambda k, v, t=target: gh_command( + "PUT", f"{t}/defaults", {k: v}) if k == "default_for_new_repos" else gh_command("PATCH", t, {k: v})) + for name in sorted(set(live) - set(self.base.get("security_configurations", {}))): + target = f"{path}/{live[name]['id']}" + self.diff(target, f"security configuration {name!r} is not in the baseline", + f"record it, or {gh_command('DELETE', target)}") + + def check_repositories(self, live_repos): + org, wanted = self.org, self.base.get("repositories", {}) + for name in sorted(set(live_repos) - set(wanted)): + self.diff(f"{org}/{name}", "repository is not classified", 'add it under "repositories" with a profile') + for name in sorted(set(wanted) - set(live_repos)): + self.diff(f"{org}/{name}", "repository is in the baseline only", "remove it from the baseline") + names = sorted(set(live_repos) & set(wanted)) + reads = [] + for name in names: + branch = urllib.parse.quote(live_repos[name]["default_branch"], safe="") + base = f"repos/{org}/{name}" + reads += [(base + p, False) for p in ("", "/code-security-configuration", "/environments?per_page=100")] + reads += [(base + p, True) for p in (f"/rules/branches/{branch}?per_page=100", "/hooks?per_page=100", + "/rulesets?includes_parents=true&per_page=100", "/teams?per_page=100")] + self.gh.prefetch(reads) + grants, unread = {}, set() + for name in names: + profile, entry = profile_for(self.base, wanted[name]), wanted[name] + repo = self.read(f"{org}/{name}", f"repos/{org}/{name}") + if repo is None: + continue + self.compare_fields(f"{org}/{name}", profile.get("repository", {}), repo, + lambda key, value, n=name: gh_command("PATCH", f"repos/{org}/{n}", {key: value}), + None if (repo.get("permissions") or {}).get("admin") else "admin on the repository") + if "branch_rules" in profile: + self.check_branch_rules(name, repo, profile["branch_rules"], entry.get("checks", [])) + if "tag_rules" in profile: + self.check_tag_rules(name, profile["tag_rules"]) + if "security_configuration" in profile: + self.check_attachment(name, repo, profile["security_configuration"]) + teams = self.read(f"{org}/{name} teams", f"repos/{org}/{name}/teams?per_page=100", True) + unread |= {name} if teams is None else set() + for team in teams or []: + grants.setdefault(team["slug"], {})[name] = team["permission"] + self.check_environments(name) + self.check_webhooks(f"{org}/{name}", f"repos/{org}/{name}/hooks", f"https://github.com/{org}/{name}", + self.recorded.get("webhooks", {}).get(name, [])) + self.check_teams(grants, set(names) - unread) + self.check_webhooks(f"orgs/{org}", f"orgs/{org}/hooks", f"https://github.com/organizations/{org}", + self.recorded.get("organization_webhooks", [])) + + def rule_source(self, name, rule): + """Where an effective rule comes from, and its fix: a command only for a repository ruleset.""" + rid, kind = rule.get("ruleset_id"), str(rule.get("ruleset_source_type") or "unknown").lower() + if kind != "repository": + return f"{kind} ruleset {rid}", SEE_RULESETS + path = f"repos/{self.org}/{name}/rulesets/{rid}" + return f"repository ruleset {rid}", (f"change the rule in {path} (read it, edit, PUT it back with --input -), " + f"or {gh_command('DELETE', path)}") + + def check_branch_rules(self, name, repo, want_rules, checks): + where = f"{self.org}/{name} {repo['default_branch']}" + branch = urllib.parse.quote(repo["default_branch"], safe="") + live = self.read(where, f"repos/{self.org}/{name}/rules/branches/{branch}?per_page=100", True) + if live is None: + return + for rule_type in sorted(set(want_rules) - {r["type"] for r in live}): + self.diff(where, f"no {rule_type} rule", SEE_RULESETS) + required = [r for r in live if r["type"] == "required_status_checks"] + for rule in (r for r in live if r not in required or r["type"] in want_rules): # from any ruleset + source, fix = self.rule_source(name, rule) + if rule["type"] not in want_rules: + self.diff(where, f"unexpected {rule['type']} rule from {source}", fix) + elif not matches(want_rules[rule["type"]], rule.get("parameters", {})): + self.diff(where, f"{rule['type']} rule from {source} is not {show(want_rules[rule['type']])}", fix) + app = self.base.get("check_integration_id") + have = sorted({(c["context"], c.get("integration_id") if app else None) for r in required + for c in r["parameters"]["required_status_checks"]}, key=str) + want = sorted({(c, app) for c in checks}, key=str) + if have != want: + sources = ", ".join(sorted({self.rule_source(name, r)[0] for r in required})) or "no ruleset" + self.diff(where, f"required checks are {show([c for c, _ in have])}" + f"{' (or from another app)' if [c for c, _ in have] == [c for c, _ in want] else ''} " + f"from {sources}, baseline {show(checks)}", SEE_RULESETS) + + def check_tag_rules(self, name, want): + where, base = f"{self.org}/{name} tags {want['ref']}", f"repos/{self.org}/{name}/rulesets" + listing = self.read(where, f"{base}?includes_parents=true&per_page=100", True) + if listing is None: + return + found, excluded = set(), [] + for item in (i for i in listing if i.get("target") == "tag" and i.get("enforcement") == "active"): + body = self.read(where, f"{base}/{item['id']}?includes_parents=true") + if body is None: # an unreadable body is "could not verify", not missing rules + return + ref_name = (body.get("conditions") or {}).get("ref_name", {}) + if {want["ref"], "~ALL"} & set(ref_name.get("include", [])): # a ruleset with ref excludes does not count + excluded += [str(item["id"])] if ref_name.get("exclude") else [] + found |= set() if ref_name.get("exclude") else {r["type"] for r in body["rules"]} + note = f" (ruleset {', '.join(excluded)} has ref excludes, so it does not count)" if excluded else "" + for rule_type in want["rules"]: + if rule_type not in found: + self.diff(where, f"no {rule_type} rule{note}", SEE_RULESETS) + + def check_attachment(self, name, repo, want): + where = f"{self.org}/{name}" + live = self.read(f"{where} security configuration", f"repos/{self.org}/{name}/code-security-configuration") + if live is None: + return + have, status = (live.get("configuration") or {}).get("name"), live.get("status") + if have != want or status not in ("attached", "enforced"): + cid = self.config_ids.get(want, "ID") + self.diff(where, f"security configuration is {have!r} ({status}), baseline {want!r}", + gh_command("POST", f"orgs/{self.org}/code-security/configurations/{cid}/attach", + {"scope": "selected", "selected_repository_ids[]": repo["id"]})) + + def check_environments(self, name): + path = f"repos/{self.org}/{name}/environments" + live = self.read(f"{self.org}/{name} environments", f"{path}?per_page=100") + undo = lambda method: lambda env: gh_command(method, f"{path}/{urllib.parse.quote(env, safe='')}") # noqa + if live is not None: + self.names(f"{self.org}/{name}", "environment", {e["name"] for e in live.get("environments", [])}, + self.recorded.get("environments", {}).get(name, []), undo("DELETE"), undo("PUT")) + + def check_webhooks(self, where, hook_path, page, recorded): + live = self.read(f"{where} webhooks", f"{hook_path}?per_page=100", True) + if live is None: + return + hosts = {} + for hook in live: + url = (hook.get("config") or {}).get("url") or "" + hosts.setdefault(urllib.parse.urlsplit(url).hostname or hook.get("name", "?"), []).append(hook["id"]) + delete = lambda host: " && ".join(gh_command("DELETE", f"{hook_path}/{hid}") for hid in hosts[host]) # noqa + self.names(where, "webhook to", hosts, recorded, delete, lambda host: f"recreate it at {page}/settings/hooks") + + def check_teams(self, grants, checked): + """Team names (organization read) and their grants on the repositories read in `checked`.""" + org, recorded = self.org, self.recorded.get("teams", {}) + teams = self.read("teams", f"orgs/{org}/teams?per_page=100", True) + live = set(recorded) if teams is None else {t["slug"] for t in teams} + self.names(f"orgs/{org}", "team", live, recorded, lambda team: gh_command("DELETE", f"orgs/{org}/teams/{team}"), + lambda team: gh_command("POST", f"orgs/{org}/teams", {"name": team})) + for team in sorted((set(grants) | set(recorded)) - (live ^ set(recorded))): # new or gone teams: reported above + have, want = grants.get(team, {}), recorded.get(team, {}) + for repo in sorted((set(have) | set(want)) & checked): + if have.get(repo) != want.get(repo): + path = f"orgs/{org}/teams/{team}/repos/{org}/{repo}" + undo = gh_command(*("PUT", path, {"permission": want[repo]}) if repo in want else ("DELETE", path)) + self.changed(f"{org}/{repo}", f"team {team!r} has {have.get(repo, 'no access')}, " + f"recorded {want.get(repo, 'no access')}", undo) + + def check_apps(self): + live = self.read("installed apps", f"orgs/{self.org}/installations?per_page=100") + if live is None: + return + have = {i["app_slug"]: i["repository_selection"] for i in live.get("installations", [])} + recorded, page = self.recorded.get("apps", {}), f"https://github.com/organizations/{self.org}/settings" + for app in sorted(set(have) | set(recorded)): + if have.get(app) != recorded.get(app): + self.changed(f"orgs/{self.org}", f"app {app!r} reaches {have.get(app, 'nothing')} repositories, " + f"recorded {recorded.get(app, 'not installed')}", f"change it at {page}/installations") + + +def run(argv, github): + parser = argparse.ArgumentParser(description="Read-only check of a GitHub organization against a baseline file.") + parser.error = lambda message: parser.exit(3, f"{parser.format_usage()}check_settings.py: {message}\n") # not 2 + parser.add_argument("baseline", help="the organization's baseline JSON file") + parser.add_argument("--validate", action="store_true", help="only check the baseline file, offline") + parser.add_argument("--ruleset", metavar="NAME", help="print the baseline body of one organization ruleset") + args = parser.parse_args(argv) + try: + with open(args.baseline, encoding="utf-8") as stream: + base = json.load(stream) + except (OSError, ValueError) as error: + print(f"{args.baseline}: cannot read the baseline: {error}", file=sys.stderr) + return 3 + errors = validate(base) + for error in errors: + print(f"{args.baseline}: {error}", file=sys.stderr) + if errors or args.validate: + return 3 if errors else 0 + if args.ruleset: + body = expected_rulesets(base).get(args.ruleset) + print(json.dumps(body, indent=2, sort_keys=True) if body else f"no ruleset {args.ruleset!r} in {args.baseline}", + file=sys.stdout if body else sys.stderr) + return 0 if body else 3 + checker = Checker(base, github or GitHub(), sys.argv[0] if argv is None else "check_settings.py", args.baseline) + try: + checker.run() + except (CannotRun, FileNotFoundError) as error: + reason = "gh is not installed or not on PATH" if isinstance(error, FileNotFoundError) else error + print(f"could not run: {reason}", file=sys.stderr) + return 3 + lines = checker.diffs + ([""] if checker.diffs and checker.unverified else []) + checker.unverified + if lines: + print("\n".join(lines)) + return 1 if checker.diffs else 2 if checker.unverified else 0 + + +def main(argv=None, github=None): + """Exit 1 means "differences", so nothing else may end in it: any unexpected error is exit 3.""" + try: + return run(argv, github) + except Exception: # a baseline shape --validate does not know, an unexpected API response, or a bug + traceback.print_exc() + print("could not run: unexpected error (traceback above); nothing was printed", file=sys.stderr) + return 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/example-org-api.json b/tests/fixtures/example-org-api.json new file mode 100644 index 0000000..3396a1e --- /dev/null +++ b/tests/fixtures/example-org-api.json @@ -0,0 +1,920 @@ +{ + "orgs/example-org": { + "default_repository_permission": "none", + "id": 42, + "login": "example-org", + "members_can_create_repositories": false, + "members_can_create_teams": false, + "two_factor_requirement_enabled": true, + "web_commit_signoff_required": false + }, + "orgs/example-org/actions/permissions": { + "allowed_actions": "selected", + "enabled_repositories": "all", + "selected_actions_url": "https://api.github.com/organizations/42/actions/permissions/selected-actions", + "sha_pinning_required": true + }, + "orgs/example-org/actions/permissions/selected-actions": { + "github_owned_allowed": true, + "patterns_allowed": [ + "example-org/*", + "docker/login-action@*" + ], + "verified_allowed": false + }, + "orgs/example-org/actions/permissions/workflow": { + "can_approve_pull_request_reviews": false, + "default_workflow_permissions": "read" + }, + "orgs/example-org/code-security/configurations/defaults": [ + { + "configuration": { + "advanced_security": "enabled", + "code_scanning_default_setup": "enabled", + "created_at": "2026-01-01T00:00:00Z", + "dependabot_alerts": "enabled", + "description": "Security features for every repository.", + "enforcement": "enforced", + "id": 501, + "name": "Example baseline", + "secret_scanning": "enabled", + "secret_scanning_push_protection": "enabled", + "target_type": "organization", + "url": "https://api.github.com/orgs/example-org/code-security/configurations/501" + }, + "default_for_new_repos": "all" + } + ], + "orgs/example-org/code-security/configurations?per_page=100": [ + { + "enforcement": "unenforced", + "id": 17, + "name": "GitHub recommended", + "target_type": "global" + }, + { + "advanced_security": "enabled", + "code_scanning_default_setup": "enabled", + "created_at": "2026-01-01T00:00:00Z", + "dependabot_alerts": "enabled", + "description": "Security features for every repository.", + "enforcement": "enforced", + "id": 501, + "name": "Example baseline", + "secret_scanning": "enabled", + "secret_scanning_push_protection": "enabled", + "target_type": "organization", + "url": "https://api.github.com/orgs/example-org/code-security/configurations/501" + } + ], + "orgs/example-org/hooks?per_page=100": [ + { + "active": true, + "config": { + "content_type": "json", + "url": "https://hooks.example.net/github" + }, + "id": 77, + "name": "web" + } + ], + "orgs/example-org/installations?per_page=100": { + "installations": [ + { + "app_slug": "example-ci", + "id": 1, + "repository_selection": "selected", + "target_type": "Organization" + }, + { + "app_slug": "example-bot", + "id": 2, + "repository_selection": "all", + "target_type": "Organization" + } + ], + "total_count": 2 + }, + "orgs/example-org/repos?per_page=100&type=all": [ + { + "archived": false, + "default_branch": "main", + "full_name": "example-org/engine", + "id": 9001, + "name": "engine", + "visibility": "public" + }, + { + "archived": false, + "default_branch": "main", + "full_name": "example-org/docs", + "id": 9002, + "name": "docs", + "visibility": "public" + }, + { + "archived": true, + "default_branch": "main", + "full_name": "example-org/legacy", + "id": 9003, + "name": "legacy", + "visibility": "public" + } + ], + "orgs/example-org/rulesets/101": { + "bypass_actors": [], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "created_at": "2026-01-01T00:00:00Z", + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "node_id": "RRS_101", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + } + ], + "source": "example-org", + "source_type": "Organization", + "target": "branch", + "updated_at": "2026-01-01T00:00:00Z" + }, + "orgs/example-org/rulesets/102": { + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "created_at": "2026-01-01T00:00:00Z", + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "node_id": "RRS_102", + "rules": [ + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "type": "pull_request" + } + ], + "source": "example-org", + "source_type": "Organization", + "target": "branch", + "updated_at": "2026-01-01T00:00:00Z" + }, + "orgs/example-org/rulesets/103": { + "bypass_actors": [], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/tags/v[0-9]*" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "~ALL" + ], + "protected": false + } + }, + "created_at": "2026-01-01T00:00:00Z", + "enforcement": "active", + "id": 103, + "name": "Release tags", + "node_id": "RRS_103", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "update" + } + ], + "source": "example-org", + "source_type": "Organization", + "target": "tag", + "updated_at": "2026-01-01T00:00:00Z" + }, + "orgs/example-org/rulesets/104": { + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + }, + "repository_name": { + "exclude": [], + "include": [ + "engine" + ], + "protected": false + } + }, + "created_at": "2026-01-01T00:00:00Z", + "enforcement": "active", + "id": 104, + "name": "Required CI - engine", + "node_id": "RRS_104", + "rules": [ + { + "parameters": { + "do_not_enforce_on_create": false, + "required_status_checks": [ + { + "context": "Engine tests", + "integration_id": 15368 + }, + { + "context": "PR title", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": true + }, + "type": "required_status_checks" + } + ], + "source": "example-org", + "source_type": "Organization", + "target": "branch", + "updated_at": "2026-01-01T00:00:00Z" + }, + "orgs/example-org/rulesets?per_page=100": [ + { + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 103, + "name": "Release tags", + "source": "example-org", + "source_type": "Organization", + "target": "tag" + }, + { + "enforcement": "active", + "id": 104, + "name": "Required CI - engine", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + } + ], + "orgs/example-org/teams?per_page=100": [ + { + "name": "maintainers", + "permission": "pull", + "slug": "maintainers" + }, + { + "name": "readers", + "slug": "readers" + } + ], + "repos/example-org/docs": { + "allow_merge_commit": false, + "allow_rebase_merge": false, + "allow_squash_merge": true, + "archived": false, + "default_branch": "main", + "delete_branch_on_merge": true, + "full_name": "example-org/docs", + "id": 9002, + "name": "docs", + "squash_merge_commit_message": "PR_BODY", + "squash_merge_commit_title": "PR_TITLE", + "visibility": "public" + }, + "repos/example-org/docs/code-security-configuration": { + "configuration": { + "id": 501, + "name": "Example baseline" + }, + "status": "enforced" + }, + "repos/example-org/docs/environments?per_page=100": { + "environments": [], + "total_count": 0 + }, + "repos/example-org/docs/hooks?per_page=100": [], + "repos/example-org/docs/rules/branches/main?per_page=100": [ + { + "ruleset_id": 101, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "deletion" + }, + { + "ruleset_id": 101, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "non_fast_forward" + }, + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "ruleset_id": 102, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "pull_request" + } + ], + "repos/example-org/docs/rulesets/101?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/docs/rulesets/102?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "rules": [ + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "type": "pull_request" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/docs/rulesets/103?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/tags/v[0-9]*" + ] + } + }, + "enforcement": "active", + "id": 103, + "name": "Release tags", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "update" + } + ], + "source_type": "Organization", + "target": "tag" + }, + "repos/example-org/docs/rulesets?includes_parents=true&per_page=100": [ + { + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 103, + "name": "Release tags", + "source": "example-org", + "source_type": "Organization", + "target": "tag" + } + ], + "repos/example-org/docs/teams?per_page=100": [ + { + "permission": "maintain", + "slug": "maintainers" + } + ], + "repos/example-org/engine": { + "allow_merge_commit": false, + "allow_rebase_merge": false, + "allow_squash_merge": true, + "archived": false, + "default_branch": "main", + "delete_branch_on_merge": true, + "full_name": "example-org/engine", + "id": 9001, + "name": "engine", + "squash_merge_commit_message": "PR_BODY", + "squash_merge_commit_title": "PR_TITLE", + "visibility": "public" + }, + "repos/example-org/engine/code-security-configuration": { + "configuration": { + "id": 501, + "name": "Example baseline" + }, + "status": "enforced" + }, + "repos/example-org/engine/environments?per_page=100": { + "environments": [ + { + "id": 1, + "name": "release" + } + ], + "total_count": 1 + }, + "repos/example-org/engine/hooks?per_page=100": [ + { + "config": { + "url": "https://ci.example.com/hook" + }, + "id": 88, + "name": "web" + } + ], + "repos/example-org/engine/rules/branches/main?per_page=100": [ + { + "ruleset_id": 101, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "deletion" + }, + { + "ruleset_id": 101, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "non_fast_forward" + }, + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "ruleset_id": 102, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "pull_request" + }, + { + "parameters": { + "do_not_enforce_on_create": false, + "required_status_checks": [ + { + "context": "Engine tests", + "integration_id": 15368 + }, + { + "context": "PR title", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": true + }, + "ruleset_id": 104, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "required_status_checks" + } + ], + "repos/example-org/engine/rulesets/101?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/engine/rulesets/102?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "rules": [ + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "type": "pull_request" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/engine/rulesets/103?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/tags/v[0-9]*" + ] + } + }, + "enforcement": "active", + "id": 103, + "name": "Release tags", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "update" + } + ], + "source_type": "Organization", + "target": "tag" + }, + "repos/example-org/engine/rulesets/104?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 104, + "name": "Required CI - engine", + "rules": [ + { + "parameters": { + "do_not_enforce_on_create": false, + "required_status_checks": [ + { + "context": "Engine tests", + "integration_id": 15368 + }, + { + "context": "PR title", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": true + }, + "type": "required_status_checks" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/engine/rulesets?includes_parents=true&per_page=100": [ + { + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 103, + "name": "Release tags", + "source": "example-org", + "source_type": "Organization", + "target": "tag" + }, + { + "enforcement": "active", + "id": 104, + "name": "Required CI - engine", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + } + ], + "repos/example-org/engine/teams?per_page=100": [ + { + "permission": "maintain", + "slug": "maintainers" + } + ], + "repos/example-org/legacy": { + "allow_merge_commit": true, + "allow_rebase_merge": true, + "allow_squash_merge": true, + "archived": true, + "default_branch": "main", + "delete_branch_on_merge": false, + "full_name": "example-org/legacy", + "id": 9003, + "name": "legacy", + "squash_merge_commit_message": "PR_BODY", + "squash_merge_commit_title": "PR_TITLE", + "visibility": "public" + }, + "repos/example-org/legacy/code-security-configuration": { + "configuration": { + "id": 501, + "name": "Example baseline" + }, + "status": "enforced" + }, + "repos/example-org/legacy/environments?per_page=100": { + "environments": [], + "total_count": 0 + }, + "repos/example-org/legacy/hooks?per_page=100": [], + "repos/example-org/legacy/rules/branches/main?per_page=100": [ + { + "ruleset_id": 101, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "deletion" + }, + { + "ruleset_id": 101, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "non_fast_forward" + }, + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "ruleset_id": 102, + "ruleset_source": "example-org", + "ruleset_source_type": "Organization", + "type": "pull_request" + } + ], + "repos/example-org/legacy/rulesets/101?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/legacy/rulesets/102?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "rules": [ + { + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "dismiss_stale_reviews_on_push": false, + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + }, + "type": "pull_request" + } + ], + "source_type": "Organization", + "target": "branch" + }, + "repos/example-org/legacy/rulesets/103?includes_parents=true": { + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "refs/tags/v[0-9]*" + ] + } + }, + "enforcement": "active", + "id": 103, + "name": "Release tags", + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "update" + } + ], + "source_type": "Organization", + "target": "tag" + }, + "repos/example-org/legacy/rulesets?includes_parents=true&per_page=100": [ + { + "enforcement": "active", + "id": 101, + "name": "Default branch integrity", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 102, + "name": "Changes through pull requests", + "source": "example-org", + "source_type": "Organization", + "target": "branch" + }, + { + "enforcement": "active", + "id": 103, + "name": "Release tags", + "source": "example-org", + "source_type": "Organization", + "target": "tag" + } + ], + "repos/example-org/legacy/teams?per_page=100": [] +} diff --git a/tests/fixtures/example-org-baseline.json b/tests/fixtures/example-org-baseline.json new file mode 100644 index 0000000..bce63cc --- /dev/null +++ b/tests/fixtures/example-org-baseline.json @@ -0,0 +1,273 @@ +{ + "organization": "example-org", + "notes": [ + "Invented test data for check_settings.py. It describes no real organization." + ], + "organization_settings": { + "default_repository_permission": "none", + "members_can_create_repositories": false, + "two_factor_requirement_enabled": true, + "members_can_create_teams": false + }, + "actions": { + "permissions": { + "enabled_repositories": "all", + "allowed_actions": "selected", + "sha_pinning_required": true + }, + "selected_actions": { + "github_owned_allowed": true, + "verified_allowed": false, + "patterns_allowed": [ + "docker/login-action@*", + "example-org/*" + ] + }, + "workflow": { + "default_workflow_permissions": "read", + "can_approve_pull_request_reviews": false + } + }, + "rulesets": { + "Default branch integrity": { + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "include": [ + "~DEFAULT_BRANCH" + ], + "exclude": [] + }, + "repository_name": { + "include": [ + "~ALL" + ], + "exclude": [], + "protected": false + } + }, + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + } + ], + "bypass_actors": [] + }, + "Changes through pull requests": { + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "include": [ + "~DEFAULT_BRANCH" + ], + "exclude": [] + }, + "repository_name": { + "include": [ + "~ALL" + ], + "exclude": [], + "protected": false + } + }, + "rules": [ + { + "type": "pull_request", + "parameters": { + "allowed_merge_methods": [ + "squash" + ], + "required_approving_review_count": 0, + "required_review_thread_resolution": true, + "dismiss_stale_reviews_on_push": false, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_reviewers": [] + } + } + ], + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ] + }, + "Release tags": { + "target": "tag", + "enforcement": "active", + "conditions": { + "ref_name": { + "include": [ + "refs/tags/v[0-9]*" + ], + "exclude": [] + }, + "repository_name": { + "include": [ + "~ALL" + ], + "exclude": [], + "protected": false + } + }, + "rules": [ + { + "type": "deletion" + }, + { + "type": "non_fast_forward" + }, + { + "type": "update" + } + ], + "bypass_actors": [] + } + }, + "required_check_ruleset": { + "name": "Required CI - {repository}", + "target": "branch", + "enforcement": "active", + "conditions": { + "ref_name": { + "include": [ + "~DEFAULT_BRANCH" + ], + "exclude": [] + }, + "repository_name": { + "include": [ + "{repository}" + ], + "exclude": [], + "protected": false + } + }, + "rules": [ + { + "type": "required_status_checks", + "parameters": { + "do_not_enforce_on_create": false, + "strict_required_status_checks_policy": true + } + } + ], + "bypass_actors": [ + { + "actor_id": null, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ] + }, + "check_integration_id": 15368, + "security_configurations": { + "Example baseline": { + "description": "Security features for every repository.", + "advanced_security": "enabled", + "secret_scanning": "enabled", + "secret_scanning_push_protection": "enabled", + "dependabot_alerts": "enabled", + "code_scanning_default_setup": "enabled", + "enforcement": "enforced", + "default_for_new_repos": "all" + } + }, + "profiles": { + "standard": { + "repository": { + "archived": false, + "allow_squash_merge": true, + "allow_merge_commit": false, + "allow_rebase_merge": false, + "delete_branch_on_merge": true, + "squash_merge_commit_title": "PR_TITLE", + "squash_merge_commit_message": "PR_BODY" + }, + "branch_rules": { + "deletion": {}, + "non_fast_forward": {}, + "pull_request": { + "allowed_merge_methods": [ + "squash" + ], + "required_review_thread_resolution": true, + "dismiss_stale_reviews_on_push": false, + "require_code_owner_review": false, + "require_last_push_approval": false, + "required_approving_review_count": 0, + "required_reviewers": [] + } + }, + "tag_rules": { + "ref": "refs/tags/v[0-9]*", + "rules": [ + "deletion", + "non_fast_forward", + "update" + ] + }, + "security_configuration": "Example baseline" + }, + "archived": { + "repository": { + "archived": true + } + } + }, + "repositories": { + "docs": { + "profile": "standard", + "checks": [] + }, + "engine": { + "profile": "standard", + "checks": [ + "Engine tests", + "PR title" + ] + }, + "legacy": { + "profile": "archived" + } + }, + "recorded": { + "teams": { + "maintainers": { + "docs": "maintain", + "engine": "maintain" + }, + "readers": {} + }, + "apps": { + "example-bot": "all", + "example-ci": "selected" + }, + "environments": { + "engine": [ + "release" + ] + }, + "webhooks": { + "engine": [ + "ci.example.com" + ] + }, + "organization_webhooks": [ + "hooks.example.net" + ] + }, + "reminders": [ + { + "item": "App example-ci key", + "note": "Rotate it around 2027-01-01. A note only; nothing fails on it." + } + ] +} diff --git a/tests/test_check_settings.py b/tests/test_check_settings.py new file mode 100644 index 0000000..1d12799 --- /dev/null +++ b/tests/test_check_settings.py @@ -0,0 +1,460 @@ +"""Offline tests for check_settings.py. Fixtures are invented; no network.""" + +import contextlib +import copy +import io +import json +import pathlib +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = pathlib.Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +import check_settings # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" +ORG = "example-org" +POINTER = check_settings.SEE_RULESETS + + +def load(name): + return json.loads((FIXTURES / name).read_text(encoding="utf-8")) + + +class Recorded: + """A fake `gh api` reading recorded responses; `{"__error__": text}` fails the read.""" + + def __init__(self, api): + self.api, self.calls = api, [] + + def __call__(self, path, paginate): + self.calls.append(path) + if path not in self.api: + raise check_settings.Unverifiable(f"no fixture for {path} (HTTP 404)") + value = self.api[path] + if isinstance(value, dict) and "__error__" in value: + raise check_settings.Unverifiable(value["__error__"]) + return copy.deepcopy(value) + + +class CheckTest(unittest.TestCase): + def setUp(self): + self.api, self.base = load("example-org-api.json"), load("example-org-baseline.json") + + def run_check(self): + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / "baseline.json" + path.write_text(json.dumps(self.base), encoding="utf-8") + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = check_settings.main([str(path)], github=check_settings.GitHub(Recorded(self.api))) + return code, out.getvalue().splitlines() + + def assertLine(self, lines, *parts): + if not any(all(part in line for part in parts) for line in lines): + self.fail(f"no line with {parts!r} in:\n" + "\n".join(lines)) + + def test_clean_organization_prints_nothing(self): + self.assertEqual(self.run_check(), (0, [])) + + def test_policy_differences_print_fix_commands(self): + self.api[f"repos/{ORG}/engine"]["allow_merge_commit"] = True + self.api[f"orgs/{ORG}/actions/permissions"]["sha_pinning_required"] = False + self.api[f"orgs/{ORG}"]["members_can_create_teams"] = True + self.api[f"orgs/{ORG}"]["default_repository_permission"] = "read" + self.api[f"orgs/{ORG}/rulesets/103"]["rules"].pop() + self.api[f"repos/{ORG}/docs/rulesets/103?includes_parents=true"]["rules"].pop() + self.api[f"repos/{ORG}/docs/code-security-configuration"] = None + self.api[f"repos/{ORG}/docs/rules/branches/main?per_page=100"].append( + {"type": "merge_queue", "ruleset_id": 999, "ruleset_source_type": "Repository", "parameters": {}}) + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine: allow_merge_commit is true, baseline false", + f"fix: gh api --method PATCH repos/{ORG}/engine -F allow_merge_commit=false") + self.assertLine(lines, "sha_pinning_required is false", f"gh api --method PUT orgs/{ORG}/actions/permissions " + "-f enabled_repositories=all -f allowed_actions=selected -F sha_pinning_required=true") + self.assertLine(lines, "members_can_create_teams", "settings/member_privileges") + self.assertLine(lines, f"gh api --method PATCH orgs/{ORG} -f default_repository_permission=none") + self.assertLine(lines, "'Release tags' differs in rules", "--ruleset 'Release tags'", + f"gh api --method PUT orgs/{ORG}/rulesets/103 --input -") + self.assertLine(lines, f"{ORG}/docs tags refs/tags/v[0-9]*: no update rule", POINTER) + self.assertLine(lines, f"{ORG}/docs: security configuration is None", "configurations/501/attach", + "-f scope=selected -F 'selected_repository_ids[]=9002'") + self.assertLine(lines, "unexpected merge_queue rule from repository ruleset 999", + f"gh api --method DELETE repos/{ORG}/docs/rulesets/999") + self.assertFalse(any(line.startswith("could not verify") for line in lines)) + + def test_required_checks_compare_names_and_app(self): + rules = self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] + next(r for r in rules if r["type"] == "required_status_checks")["parameters"]["required_status_checks"][0][ + "integration_id"] = 1 + self.base["repositories"]["docs"]["checks"] = ["Docs build"] + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine main: required checks", "(or from another app) from organization " + "ruleset 104", POINTER) + self.assertLine(lines, f"{ORG}/docs main: required checks are [] from no ruleset, baseline [\"Docs build\"]", + POINTER) + self.assertLine(lines, "ruleset 'Required CI - docs' is missing", "--ruleset 'Required CI - docs'", + f"gh api --method POST orgs/{ORG}/rulesets --input -") + + def test_unclassified_and_unknown_objects_fail(self): + repos = self.api[f"orgs/{ORG}/repos?per_page=100&type=all"] + repos.append(dict(repos[0], name="sandbox", id=9004)) + self.api[f"orgs/{ORG}/rulesets?per_page=100"].append({"id": 105, "name": "Hand-made", "target": "branch"}) + self.api[f"orgs/{ORG}/rulesets/105"] = {"id": 105, "name": "Hand-made", "target": "branch"} + del self.base["repositories"]["legacy"] + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/sandbox: repository is not classified") + self.assertLine(lines, f"{ORG}/legacy: repository is not classified") + self.assertLine(lines, "ruleset 'Hand-made' is not in the baseline", f"DELETE orgs/{ORG}/rulesets/105") + + def test_recorded_names_report_changes(self): + self.api[f"repos/{ORG}/engine/environments?per_page=100"]["environments"].append({"name": "staging"}) + self.api[f"repos/{ORG}/docs/hooks?per_page=100"].append({"id": 5, "config": {"url": "https://x.example.org/h"}}) + self.api[f"repos/{ORG}/docs/teams?per_page=100"][0]["permission"] = "admin" + self.api[f"orgs/{ORG}/installations?per_page=100"]["installations"][0]["repository_selection"] = "all" + self.api[f"orgs/{ORG}/teams?per_page=100"].append({"slug": "visitors"}) + self.base["recorded"]["environments"]["docs"] = ["preview"] + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, "environment 'staging' is new", "changed since last recorded", + f"DELETE repos/{ORG}/engine/environments/staging") + self.assertLine(lines, "environment 'preview' is gone", f"PUT repos/{ORG}/docs/environments/preview") + self.assertLine(lines, "webhook to 'x.example.org' is new", f"DELETE repos/{ORG}/docs/hooks/5") + self.assertLine(lines, "team 'maintainers' has admin, recorded maintain", + f"PUT orgs/{ORG}/teams/maintainers/repos/{ORG}/docs -f permission=maintain") + self.assertLine(lines, "app 'example-ci' reaches all repositories, recorded selected", "settings/installations") + self.assertLine(lines, "team 'visitors' is new") + + def test_without_admin_org_only_unverifiable(self): + for path in list(self.api): + if path.startswith(f"orgs/{ORG}") and path != f"orgs/{ORG}/repos?per_page=100&type=all": + self.api[path] = {"__error__": "Forbidden (HTTP 403)"} + code, lines = self.run_check() + self.assertEqual(code, 2) + self.assertTrue(lines and all(line.startswith("could not verify: ") for line in lines), lines) + self.assertLine(lines, "organization rulesets", "HTTP 403") + self.assertLine(lines, "orgs/example-org/actions/permissions") + + def test_without_admin_org_repository_checks_still_run(self): + for path in list(self.api): + if path.startswith(f"orgs/{ORG}") and path != f"orgs/{ORG}/repos?per_page=100&type=all": + self.api[path] = {"__error__": "Forbidden (HTTP 403)"} + self.api[f"repos/{ORG}/engine"]["delete_branch_on_merge"] = False + self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] = [] + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, "delete_branch_on_merge is false") + self.assertLine(lines, f"{ORG}/engine main: no deletion rule", POINTER) + self.assertLine(lines, "could not verify: teams") + + def test_rules_from_an_organization_ruleset_are_always_reported_and_carry_no_command(self): + """Its own line has the command when it differs; the per-repository lines never guess one.""" + extra = {"type": "merge_queue", "ruleset_id": 102, "ruleset_source_type": "Organization", "parameters": {}} + self.api[f"orgs/{ORG}/rulesets/102"]["rules"].append({"type": "merge_queue", "parameters": {}}) + for repo in ("docs", "engine"): + self.api[f"repos/{ORG}/{repo}/rules/branches/main?per_page=100"].append(dict(extra)) + states = {"read": None, "unreadable": {f"orgs/{ORG}/rulesets/102"}, "no admin:org": { + path for path in self.api if path.startswith(f"orgs/{ORG}") and "/repos?" not in path}} + for state, failing in states.items(): + for path in failing or (): + self.api[path] = {"__error__": "Forbidden (HTTP 403)"} + with self.subTest(state=state): + code, lines = self.run_check() + self.assertEqual(code, 1) + for repo in ("docs", "engine"): + self.assertLine(lines, f"{ORG}/{repo} main: unexpected merge_queue rule from organization " + "ruleset 102", f"fix: {POINTER}") + own = [line for line in lines if line.startswith(f"orgs/{ORG}/rulesets/102: ")] + self.assertEqual(len(own), 1 if state == "read" else 0, lines) + if own: + self.assertIn(f"gh api --method PUT orgs/{ORG}/rulesets/102 --input -", own[0]) + else: + self.assertLine(lines, "could not verify: ", "HTTP 403") + self.assertFalse(any("DELETE" in line for line in lines), lines) + + def test_repository_pull_request_rule_is_compared_in_full(self): + rules = self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] + other = copy.deepcopy(next(r for r in rules if r["type"] == "pull_request")) + other.update(ruleset_id=999, ruleset_source_type="Repository") + other["parameters"].update(require_code_owner_review=True, required_reviewers=[{"reviewer": {"id": 1}}]) + rules.append(other) + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine main: pull_request rule from repository ruleset 999 is not", + f"repos/{ORG}/engine/rulesets/999", f"gh api --method DELETE repos/{ORG}/engine/rulesets/999") + # The organization ruleset already matches, so its PUT would not fix this. + self.assertFalse(any("orgs/example-org/rulesets" in line for line in lines), lines) + + def test_required_status_checks_parameters_a_profile_names_are_compared(self): + self.base["profiles"]["standard"]["branch_rules"]["required_status_checks"] = { + "strict_required_status_checks_policy": True} + rules = self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] + rules.append({"type": "required_status_checks", "ruleset_id": 999, "ruleset_source_type": "Repository", + "parameters": {"strict_required_status_checks_policy": False, "required_status_checks": [ + {"context": "Engine tests", "integration_id": 15368}]}}) + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine main: required_status_checks rule from repository ruleset 999 is not", + f"repos/{ORG}/engine/rulesets/999") + + def test_a_rule_of_unknown_origin_gets_no_command(self): + self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"].append({"type": "merge_queue"}) + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine main: unexpected merge_queue rule from unknown ruleset None", POINTER) + self.assertFalse(any("gh api" in line for line in lines), lines) + + def test_differing_rule_from_an_unrecorded_organization_ruleset(self): + rules = self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] + other = copy.deepcopy(next(r for r in rules if r["type"] == "pull_request")) + other.update(ruleset_id=555, ruleset_source_type="Organization") + other["parameters"]["require_code_owner_review"] = True + rules.append(other) + self.api[f"orgs/{ORG}/rulesets?per_page=100"].append({"id": 555, "name": "Extra review", "target": "branch"}) + self.api[f"orgs/{ORG}/rulesets/555"] = {"id": 555, "name": "Extra review", "target": "branch"} + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine main: pull_request rule from organization ruleset 555 is not", + f"fix: {POINTER}") + self.assertLine(lines, "ruleset 'Extra review' is not in the baseline", f"DELETE orgs/{ORG}/rulesets/555") + # The matching baseline ruleset 102 would be a no-op PUT, so it is never the fix. + self.assertFalse(any("rulesets/102" in line for line in lines), lines) + + def test_recorded_organization_ruleset_that_disagrees_with_the_profile(self): + recorded = self.base["rulesets"]["Changes through pull requests"]["rules"] + recorded[0]["parameters"]["require_code_owner_review"] = True + recorded.append({"type": "merge_queue", "parameters": {}}) + self.api[f"orgs/{ORG}/rulesets/102"]["rules"] = copy.deepcopy(recorded) # live matches its baseline body + for repo in ("docs", "engine"): + rules = self.api[f"repos/{ORG}/{repo}/rules/branches/main?per_page=100"] + next(r for r in rules if r["type"] == "pull_request")["parameters"]["require_code_owner_review"] = True + rules.append({"type": "merge_queue", "ruleset_id": 102, "ruleset_source_type": "Organization"}) + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/engine main: pull_request rule from organization ruleset 102 is not", POINTER) + self.assertLine(lines, f"{ORG}/engine main: unexpected merge_queue rule from organization ruleset 102", POINTER) + # No ruleset line and no command: a PUT would send the body that is already live. + self.assertFalse(any("gh api" in line for line in lines), lines) + + def test_unreadable_tag_ruleset_is_only_unverifiable(self): + self.api[f"repos/{ORG}/docs/rulesets/103?includes_parents=true"] = {"__error__": "Forbidden (HTTP 403)"} + code, lines = self.run_check() + self.assertEqual(code, 2) + self.assertEqual(lines, [f"could not verify: {ORG}/docs tags refs/tags/v[0-9]*: Forbidden (HTTP 403)"]) + + def test_one_timeout_keeps_the_other_differences(self): + self.api[f"orgs/{ORG}"]["members_can_create_teams"] = True + self.api[f"repos/{ORG}/engine/hooks?per_page=100"] = {"__error__": "Get ...: dial tcp: i/o timeout"} + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, "members_can_create_teams is true") + self.assertLine(lines, f"could not verify: {ORG}/engine webhooks: Get ...: dial tcp: i/o timeout") + + def test_tag_excludes_and_other_repository_rules_are_differences(self): + self.api[f"repos/{ORG}/docs/rulesets/103?includes_parents=true"]["conditions"]["ref_name"]["exclude"] = [ + "refs/tags/v1*"] + rules = self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] + other = copy.deepcopy(next(r for r in rules if r["type"] == "pull_request")) + other.update(ruleset_id=777, ruleset_source_type="Repository") + other["parameters"]["allowed_merge_methods"] = ["merge", "squash"] + rules.append(other) + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"{ORG}/docs tags refs/tags/v[0-9]*: no deletion rule", "ruleset 103 has ref excludes") + self.assertLine(lines, f"{ORG}/engine main: pull_request rule from repository ruleset 777 is not", + f"repos/{ORG}/engine/rulesets/777") + + def test_nothing_verified_exits_3(self): + def logged_out(path, paginate): + raise check_settings.CannotRun("To get started with GitHub CLI, please run: gh auth login") + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = check_settings.main([str(FIXTURES / "example-org-baseline.json")], github=check_settings.GitHub(logged_out)) + self.assertEqual((code, out.getvalue()), (3, "")) + self.assertIn("could not run: ", err.getvalue()) + self.api[f"orgs/{ORG}/repos?per_page=100&type=all"] = {"__error__": "Forbidden (HTTP 403)"} + with contextlib.redirect_stderr(err): + self.assertEqual(self.run_check()[0], 3) + self.assertIn("cannot list the repositories", err.getvalue()) + + def test_unexpected_response_exits_3(self): + rules = self.api[f"repos/{ORG}/engine/rules/branches/main?per_page=100"] + del next(r for r in rules if r["type"] == "required_status_checks")["parameters"] + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(self.run_check()[0], 3) + self.assertIn("KeyError: 'parameters'", err.getvalue()) + self.assertIn("could not run: unexpected error", err.getvalue()) + + def test_a_crash_exits_3_and_never_1(self): + """Exit 1 means "differences", so a baseline shape --validate does not know must not end in it.""" + self.base["check_integration_id"] = {"not": "an id"} # passes --validate, breaks the comparison + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(self.run_check(), (3, [])) + self.assertIn("TypeError", err.getvalue()) + self.assertIn("could not run: unexpected error", err.getvalue()) + + def test_hidden_organization_fields_are_unverifiable(self): + for key in ("members_can_create_repositories", "members_can_create_teams", "two_factor_requirement_enabled"): + del self.api[f"orgs/{ORG}"][key] # what a member without admin:org sees + code, lines = self.run_check() + self.assertEqual(code, 2) + self.assertEqual(lines, [f"could not verify: orgs/{ORG}: members_can_create_repositories, " + "members_can_create_teams, two_factor_requirement_enabled not visible to this login " + "(needs admin:org, or a field name is misspelled)"]) + + def test_misspelled_fields_are_baseline_errors_when_the_login_sees_everything(self): + self.base["organization_settings"]["members_can_create_team"] = False + self.base["profiles"]["standard"]["repository"]["allow_squash_merges"] = True + self.api[f"repos/{ORG}/engine"]["permissions"] = {"admin": True} + self.base["security_configurations"]["Example baseline"]["dependabot_alert"] = "enabled" + code, lines = self.run_check() + self.assertEqual(code, 1) + self.assertLine(lines, f"orgs/{ORG}: members_can_create_team is not in the API response", + "correct the field name in the baseline") + self.assertLine(lines, f"{ORG}/engine: allow_squash_merges is not in the API response") + self.assertLine(lines, "(Example baseline): dependabot_alert is not in the API response") + # Without admin on the repository the field may be hidden, so it is unverifiable, never blamed on admin:org. + self.assertLine(lines, f"could not verify: {ORG}/docs: allow_squash_merges not visible to this login " + "(needs admin on the repository, or a field name is misspelled)") + + +class BaselineTest(unittest.TestCase): + def setUp(self): + self.base = load("example-org-baseline.json") + + def test_real_baselines_are_valid(self): + for path in sorted((ROOT / "baselines").glob("*.json")): + with self.subTest(path=path.name): + self.assertEqual(check_settings.validate(json.loads(path.read_text(encoding="utf-8"))), []) + + def test_at_most_three_profiles(self): + for name in ("a", "b"): + self.base["profiles"][name] = {} + self.assertIn("1 to 3 profiles", " ".join(check_settings.validate(self.base))) + + def test_at_most_ten_exceptions(self): + for number in range(11): + self.base["repositories"][f"r{number}"] = {"profile": "standard", "why": "test", + "overrides": {"tag_rules": {"ref": "x", "rules": []}}} + self.assertIn("11 per-repository exceptions", " ".join(check_settings.validate(self.base))) + + def test_rejects_unknown_profile_key_and_unexplained_override(self): + self.base["repositories"]["docs"]["profile"] = "missing" + self.base["repositories"]["engine"]["overrides"] = {"security_configuration": "Example baseline"} + self.base["profiles"]["archived"]["expires"] = "2026-01-01" + errors = " ".join(check_settings.validate(self.base)) + self.assertIn("docs: needs a known profile", errors) + self.assertIn('engine: an override needs a "why"', errors) + self.assertIn("unknown key 'expires'", errors) + + def test_rejects_shapes_the_check_cannot_read(self): + for change, message in [(lambda b: b["profiles"]["standard"]["tag_rules"].pop("ref"), "standard: tag_rules must be"), + (lambda b: b["profiles"]["standard"].update(branch_rules=[]), "branch_rules must be"), + (lambda b: b["repositories"]["engine"].update(overrides={"repository": 1}, why="x"), + "engine override: repository must be"), + (lambda b: next(iter(b["rulesets"].values()))["rules"].append({}), + "rules with a type"), + (lambda b: b["required_check_ruleset"]["rules"][0].update(parameters=3), + "required_check_ruleset must"), + (lambda b: b["required_check_ruleset"]["rules"][0].update(parameters=[]), + "required_check_ruleset must"), + (lambda b: next(iter(b["rulesets"].values()))["rules"][0].update(parameters="x"), + "rulesets must be"), + (lambda b: b["actions"].update({"selected-actions": {}}), "actions must be"), + (lambda b: b["actions"].update(workflow=[]), "actions must be"), + (lambda b: b["recorded"].update(teams=["x"]), "recorded must be"), + (lambda b: b["recorded"].update(team={}), "recorded must be"), + (lambda b: b["security_configurations"].update(x=1), "security_configurations must be"), + (lambda b: b.update(security_configurations=3), "security_configurations must be"), + (lambda b: b.update(security_configurations=None), "security_configurations must be"), + (lambda b: b.update(security_configurations=True), "security_configurations must be"), + (lambda b: b.update(organization_settings=[]), "organization_settings must be"), + (lambda b: b["reminders"].append({"item": "x"}), "reminders must be"), + (lambda b: b["required_check_ruleset"].update(name="CI"), "required_check_ruleset must"), + (lambda b: b.pop("organization"), "organization must be"), + (lambda b: b.update(repositories=[]), "repositories must be"), + (lambda b: b["repositories"]["docs"].update(profile=["x"]), "docs: needs a known"), + (lambda b: b["repositories"]["docs"].update(overrides=3, why="x"), + "docs override: must be an object")]: + base = copy.deepcopy(self.base) + change(base) + with self.subTest(message=message): + self.assertIn(message, " ".join(check_settings.validate(base))) + + def test_ruleset_option_expands_the_template(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = check_settings.main([str(FIXTURES / "example-org-baseline.json"), "--ruleset", "Required CI - engine"]) + body = json.loads(out.getvalue()) + self.assertEqual(code, 0) + self.assertEqual(body["conditions"]["repository_name"]["include"], ["engine"]) + self.assertEqual(body["rules"][0]["parameters"]["required_status_checks"], + [{"context": "Engine tests", "integration_id": 15368}, {"context": "PR title", "integration_id": 15368}]) + + def test_bad_input_exits_3(self): + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(check_settings.main([str(FIXTURES / "missing.json")]), 3) + for argv in ([], ["x.json", "--no-such-flag"]): + with self.assertRaises(SystemExit) as raised: + check_settings.main(argv) + self.assertEqual(raised.exception.code, 3) + + +class ReadOnlyTest(unittest.TestCase): + def test_gh_is_only_called_with_get(self): + result = subprocess.CompletedProcess([], 0, stdout='[{"a": 1}]\n', stderr="") + with mock.patch.object(check_settings.subprocess, "run", return_value=result) as run: + self.assertEqual(check_settings.gh_fetch("orgs/x/repos?per_page=100", True), [{"a": 1}]) + self.assertEqual(run.call_args.args[0], ["gh", "api", "-X", "GET", "orgs/x/repos?per_page=100", "--paginate"]) + source = (ROOT / "check_settings.py").read_text(encoding="utf-8") + self.assertEqual(source.count("subprocess.run("), 1) + + def test_failed_read_names_the_missing_scope(self): + stderr = 'gh: Not Found (HTTP 404)\nThis API operation needs the "admin:org_hook" scope.\n' + result = subprocess.CompletedProcess([], 1, stdout="", stderr=stderr) + with mock.patch.object(check_settings.subprocess, "run", return_value=result): + with self.assertRaisesRegex(check_settings.Unverifiable, "lacks the admin:org_hook scope"): + check_settings.gh_fetch("orgs/x/hooks", True) + + def test_logged_out_or_offline_cannot_run(self): + for stderr in ("To get started with GitHub CLI, please run: gh auth login\n", "gh: Bad credentials (HTTP 401)\n"): + result = subprocess.CompletedProcess([], 1, stdout="", stderr=stderr) + with self.subTest(stderr=stderr), mock.patch.object(check_settings.subprocess, "run", return_value=result): + with self.assertRaises(check_settings.CannotRun): + check_settings.gh_fetch("orgs/x", False) + result = subprocess.CompletedProcess([], 1, stdout="", stderr="Get \"https://api.github.com\": dial tcp: i/o timeout\n") + with mock.patch.object(check_settings.subprocess, "run", return_value=result): + with self.assertRaises(check_settings.Unverifiable): # one slow read must not stop the run + check_settings.gh_fetch("repos/x/y/hooks", True) + + def test_pages_are_joined(self): + self.assertEqual(check_settings.parse_output('[1, 2]\n[3]\n'), [1, 2, 3]) + self.assertEqual(check_settings.parse_output('{"a": 1}'), {"a": 1}) + self.assertIsNone(check_settings.parse_output("")) + + def test_fix_commands_quote_values(self): + self.assertEqual(check_settings.gh_command("PATCH", "repos/o/r", {"a": True, "b": "x y"}), + "gh api --method PATCH repos/o/r -F a=true -f 'b=x y'") + self.assertEqual(check_settings.gh_command("PUT", "p", {"list": ["it's"]}), + "echo '{\"list\":[\"it'\"'\"'s\"]}' | gh api --method PUT p --input -") + + def test_matching_ignores_order_and_extra_fields(self): + self.assertTrue(check_settings.matches({"a": [1, 2]}, {"a": [2, 1], "b": 3})) + self.assertFalse(check_settings.matches({"a": [1]}, {"a": [1, 1]})) + self.assertFalse(check_settings.matches(False, 0)) + + def test_script_stays_small(self): + lines = (ROOT / "check_settings.py").read_text(encoding="utf-8").count("\n") + self.assertLessEqual(lines, 550, "the check is meant to stay about 500 lines; simplify before growing it") + + +if __name__ == "__main__": + unittest.main()