diff --git a/fork-ci/actions/docs-scope/docs_scope.py b/fork-ci/actions/docs-scope/docs_scope.py index 54de3c556533e..7467b4524f365 100644 --- a/fork-ci/actions/docs-scope/docs_scope.py +++ b/fork-ci/actions/docs-scope/docs_scope.py @@ -47,12 +47,12 @@ class Unclear(Exception): """Anything that makes the answer unknown: the run is full and this is the reason.""" -# Every git call: no global or system config (on a self-hosted runner an earlier job may have written ~/.gitconfig), and -# no hooks, fsmonitor command, commit-graph or replace refs from the repository. The same as +# Every git call: no global or system config or attributes (on a self-hosted runner an earlier job may have written +# ~/.gitconfig), and no hooks, fsmonitor command, commit-graph or replace refs from the repository. The same as # ../git-clean/git_clean.py's (git_clean.test.py pins them); the action's first step refused any other .git config. -GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'} +GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} GIT_SAFE = ('-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.commitGraph=false', - '--no-replace-objects') + '-c', 'core.attributesFile=/dev/null', '--no-replace-objects') def git(*args): diff --git a/fork-ci/actions/git-clean/README.md b/fork-ci/actions/git-clean/README.md index 4884489876602..8fe5838588040 100644 --- a/fork-ci/actions/git-clean/README.md +++ b/fork-ci/actions/git-clean/README.md @@ -7,14 +7,18 @@ workspace, and an earlier job may have run pull request code there. That code ca of its own steps runs pull request code. - **Self-hosted runner:** it empties the workspace (`GITHUB_WORKSPACE`), so the checkout starts from nothing, as on a - GitHub-hosted runner. Symlinks are removed as links, never followed: nothing outside the workspace is touched. + GitHub-hosted runner. Symlinks are removed as links, never followed, and the workspace's real path must be its own + path below the runner's work folder (no symlink from `RUNNER_WORKSPACE` down): nothing outside the workspace is + touched. The walk keeps one folder open at a time, so a tree of any depth is removed. - **GitHub-hosted runner:** each job starts on a fresh disk, so nothing is removed. - **Both:** it then refuses a `.git` at the checkout path whose config `actions/checkout` does not write (below). -The fork-ci actions that read a checkout with git (docs-scope, job-changed, test-select, head-check, pr-queue's -tested-base) run the same check as their first step, and every git call of theirs runs with no global or system -config (`GIT_CONFIG_GLOBAL=/dev/null`, `GIT_CONFIG_NOSYSTEM=1`) and with hooks, `core.fsmonitor`, the commit-graph and -replace refs off. tested-base writes no note instead of failing. +The fork-ci actions that read a checkout with git (docs-scope, job-changed, test-select, head-check, weakened-tests, +pr-queue's tested-base) run the same check as their first step, and every git call of theirs runs with no global or +system config or attributes (`GIT_CONFIG_GLOBAL=/dev/null`, `GIT_CONFIG_NOSYSTEM=1`, `GIT_ATTR_NOSYSTEM=1`, +`core.attributesFile=/dev/null`) and with hooks, `core.fsmonitor`, the commit-graph and replace refs off. On a refusal +the report-only actions do not fail: tested-base writes no note, weakened-tests reports `status` `refused` and replaces +its earlier comment. The others fail the step. Their `path` input must be relative to the workspace. ## Usage @@ -34,6 +38,14 @@ Put it in every job that runs on a self-hosted runner and checks out code, befor other step that writes the workspace. With several checkouts, list their paths (`path: | ...`): all of them are checked. +**Cost:** the reused `.git` is gone, so every self-hosted job fetches its checkout again from nothing (with +`fetch-depth: 0`, the whole history), as a GitHub-hosted job does. + +**Where it is not needed:** a runner whose disk does not outlive the job. In this fork that is every GitHub-hosted +job and the fleet runners `fork-tests.yml` routes jobs to: one-job (JIT) runners of trusted runs, each started from a +fresh copy that is removed afterwards (`.github/fork-fleet.md`). `fork-pr-queue.yml`'s `queue` job does reuse its +disk, but it takes no `pull_request` job and its git runs only in its own checked cache (`../pr-queue/README.md`). + | Input | Default | Meaning | | ------ | ------- | -------------------------------------------------------------------- | | `path` | `.` | the checkout path or paths (actions/checkout's `path`), one per line | @@ -42,23 +54,43 @@ No outputs. A refusal fails the step, so the checkout and every later step do no ## What is refused -- `GITHUB_WORKSPACE` that is not an absolute folder, is a symlink, is not inside `RUNNER_WORKSPACE`, or holds `HOME`, - `RUNNER_TEMP`, `RUNNER_TOOL_CACHE` or the action itself: nothing is removed. +- `GITHUB_WORKSPACE` that is not an absolute folder, is a symlink, is not inside `RUNNER_WORKSPACE` (which a + self-hosted runner always sets), has a real path other than its path below the runner's work folder (an earlier job + made `RUNNER_WORKSPACE` or a folder below it a symlink), or holds `HOME`, `RUNNER_TEMP`, `RUNNER_TOOL_CACHE` or the + action itself: nothing is removed. - A delete that fails (a file of another user, for example from a container): the step fails with the path. What is left is removed by the next run once the file can be deleted. - A checkout path that is absolute or leads outside the workspace. - `/.git` that is a file or a symlink (a worktree or submodule: git would read another repository), or that holds - `commondir`. + `commondir`, `info/grafts`, `objects/info/alternates`, `objects/info/http-alternates` or `info/attributes` (they + change a commit's parents, add another repository's objects or change how a file merges; `--no-replace-objects` + does not turn grafts off). +- `/.git/shallow` other than the one a `fetch-depth: 1` checkout writes (exactly the commit of `HEAD`): a deeper + shallow fetch (`fetch-depth: 2` or more) is refused too. Check out with `fetch-depth: 0` or `1`. - A key in `.git/config` or `.git/config.worktree` that `actions/checkout` (v4 to v7) and `git init` do not write, or one of those keys with another kind of value. Allowed: the core settings of `git init`, sparse checkout, the object format, `gc.auto 0`, `remote.origin` url/fetch/partial clone, `branch.*.remote`/`merge`, the token header, the LFS - filter exactly as `git lfs install` writes it, submodule url/active, `user.name`/`user.email`. The include + filter exactly as `git lfs install` writes it, submodule url (`https://` or `git@host:`, as for origin)/active, + `user.name`/`user.email`. The include checkout writes for its token (`includeIf.gitdir:...path`) only to a `git-credentials-.config` in `RUNNER_TEMP` that holds nothing but the token header. Everything else is refused by name, among it `core.fsmonitor`, `core.hooksPath`, `core.sshCommand` (checkout's `ssh-key` with `persist-credentials`: use `persist-credentials: false`), `core.pager`, `attr.tree`, merge/diff/filter drivers, `include.path`, other includes, `url.*.insteadOf` and `alias.*`. +The refusal says what to do either way: a key an earlier job left needs git-clean before `actions/checkout`; a key +`actions/checkout` wrote itself (`ssh-key` with `persist-credentials`, a deeper `fetch-depth`) needs another input. + +## Runs again, stops half-way, old or empty state + +- **Runs again:** an empty workspace is emptied again (nothing to do) and passes. +- **Stops half-way:** a delete that fails stops the step red, so no checkout runs on a half-empty workspace. The next + run removes what is left, at any depth. +- **Old workspace** (from before git-clean, or any earlier job): emptied like any other; a `.git` in it never reaches + a checkout. +- **Empty workspace:** passes; the actions' own check (`--require`) then refuses a missing `.git`. +- **GitHub-hosted runner:** nothing is removed; the check still runs. + ## Not covered The runner user's own files outside the workspace (`~/.gitconfig`, the system git config) are not removed: the fork-ci diff --git a/fork-ci/actions/git-clean/git_clean.py b/fork-ci/actions/git-clean/git_clean.py index 1a37b3f87fc67..a99da519e13db 100644 --- a/fork-ci/actions/git-clean/git_clean.py +++ b/fork-ci/actions/git-clean/git_clean.py @@ -11,10 +11,12 @@ before any of its own steps runs pull request code. prepare: on a self-hosted runner (RUNNER_ENVIRONMENT is not `github-hosted`) every entry of GITHUB_WORKSPACE is -removed, symlinks as links (never what they point to), so nothing outside the workspace is touched. A -GitHub-hosted runner starts each job on a fresh disk, so nothing is removed there. Then `check` runs on each path. +removed, symlinks as links (never what they point to), so nothing outside the workspace is touched. The workspace's +real path must be its path below the runner's work folder (no symlink from RUNNER_WORKSPACE down). A GitHub-hosted +runner starts each job on a fresh disk, so nothing is removed there. Then `check` runs on each path. -check: `DIR/.git` must be a folder (not a .git file or symlink, which point git elsewhere), with no `commondir`, and +check: `DIR/.git` must be a folder (not a .git file or symlink, which point git elsewhere), with no `commondir`, none +of the files that change history or objects (UNTRUSTED_FILES; `shallow` only as a depth-1 checkout writes it), and every key of its config (and config.worktree) must be one actions/checkout writes, with a value of the kind it writes (ALLOWED). The include it writes for its token is allowed only to a git-credentials file in RUNNER_TEMP that holds nothing but the token header. Anything else is refused by name: nothing is repaired in place. @@ -24,15 +26,21 @@ import argparse, os, re, stat, subprocess, sys, tempfile # Every fork-ci git call that reads a checkout runs with this environment and these flags (the actions' modules carry -# copies: git_clean.test.py pins them to these). No global or system config (an earlier job may have written -# ~/.gitconfig on a self-hosted runner); no hooks, fsmonitor command, commit-graph or replace refs from the repository. -GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'} +# copies: git_clean.test.py pins them to these). No global or system config or attributes (an earlier job may have +# written ~/.gitconfig or ~/.config/git/attributes on a self-hosted runner); no hooks, fsmonitor command, commit-graph +# or replace refs from the repository. +GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} GIT_SAFE = ('-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.commitGraph=false', - '--no-replace-objects') + '-c', 'core.attributesFile=/dev/null', '--no-replace-objects') +# Files in .git that change a commit's parents (grafts; --no-replace-objects does not turn them off), add another +# repository's objects (alternates) or change how a file merges or diffs (info/attributes). actions/checkout writes +# none of them. pr_queue.UNTRUSTED_FILES is this list and `shallow` (git_clean.test.py pins it). +UNTRUSTED_FILES = ('info/grafts', 'objects/info/alternates', 'objects/info/http-alternates', 'info/attributes') BOOL = r'(?i:true|false|yes|no|on|off|1|0)' UUID = r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' CREDENTIALS = 'credentials' # a value checked by credentials_problem, not by a pattern +URL = r'https://[^\s@]+|git@[A-Za-z0-9][A-Za-z0-9.-]*:[^\s]+' # what checkout writes; a submodule's as in .gitmodules HEADER = (r'http\.https://[^/\s]+/\.extraheader', r'AUTHORIZATION: basic [A-Za-z0-9+/=]+') # (key, value): what actions/checkout (v4 to v7) and `git init` write, as `git config --list` prints a key (section and # name lower case, a subsection as written). Everything else is refused, among it every key that runs a command or @@ -47,7 +55,7 @@ (r'extensions\.worktreeconfig', BOOL), (r'extensions\.partialclone', r'origin'), (r'gc\.auto', r'0'), - (r'remote\.origin\.url', r'https://[^\s@]+|git@[A-Za-z0-9.-]+:[^\s]+'), + (r'remote\.origin\.url', URL), (r'remote\.origin\.fetch', r'\+?refs/[^\s:]+:refs/[^\s:]+'), (r'remote\.origin\.promisor', BOOL), (r'remote\.origin\.partialclonefilter', r'[A-Za-z0-9:=+-]+'), @@ -60,7 +68,7 @@ (r'filter\.lfs\.smudge', r'git-lfs smudge -- %f'), (r'filter\.lfs\.process', r'git-lfs filter-process'), (r'filter\.lfs\.required', BOOL), - (r'submodule\.[^\s]+\.url', r'https://[^\s@]+'), + (r'submodule\.[^\s]+\.url', URL), (r'submodule\.[^\s]+\.active', BOOL), (r'user\.(name|email)', r'[^\x00-\x1f\x7f]*'), )) @@ -72,14 +80,19 @@ def say(text): print(text, flush=True) -def entries(path, env): - """[(key, value)] of one config file read by git itself (no includes followed, no other config), or raises - ValueError with git's reason.""" +def git(env, *args): + """git with the fixed settings, from an empty folder, with nothing of the runner's environment but PATH.""" with tempfile.TemporaryDirectory(prefix='git-clean-') as empty: clean = {'PATH': env.get('PATH') or os.defpath, 'LC_ALL': 'C', 'HOME': empty, 'GIT_TERMINAL_PROMPT': '0', 'GIT_CEILING_DIRECTORIES': os.path.dirname(empty), **GIT_ENV} - r = subprocess.run(['git', 'config', '--file', path, '--no-includes', '--null', '--list'], cwd=empty, - env=clean, capture_output=True, stdin=subprocess.DEVNULL) + return subprocess.run(['git', *GIT_SAFE, *args], cwd=empty, env=clean, capture_output=True, + stdin=subprocess.DEVNULL) + + +def entries(path, env): + """[(key, value)] of one config file read by git itself (no includes followed, no other config), or raises + ValueError with git's reason.""" + r = git(env, 'config', '--file', path, '--no-includes', '--null', '--list') if r.returncode != 0: raise ValueError(r.stderr.decode('utf-8', 'replace').strip()[:300] or f'git config exit {r.returncode}') out = [] @@ -133,16 +146,40 @@ def problems(repo, env, require=False): return [f'{dot} is not a folder (a .git file or symlink points git to another repository)'] if os.path.lexists(os.path.join(dot, 'commondir')): return [f'{dot} holds commondir, which makes git read the config of another repository'] - found = [] + found = [f'{dot} holds {name}, which actions/checkout never writes (it changes what history or objects git reads)' + for name in UNTRUSTED_FILES if os.path.lexists(os.path.join(dot, *name.split('/')))] for name in ('config', 'config.worktree'): path = os.path.join(dot, name) if os.path.lexists(path): found += file_problems(path, ALLOWED, env) - return found + return found or shallow_problems(dot, env) + + +def shallow_problems(dot, env): + """[why] `shallow` is not the one a fetch-depth 1 checkout writes: exactly HEAD's commit, one line. Any other cuts + history at commits git then reports as having no parents. Read only after the config passed (git runs here).""" + path = os.path.join(dot, 'shallow') + if not os.path.lexists(path): + return [] + try: + with open(path, 'rb', opener=lambda p, f: os.open(p, f | os.O_NOFOLLOW | os.O_NONBLOCK)) as f: + if not stat.S_ISREG(os.fstat(f.fileno()).st_mode): + raise OSError('not a file') + text = f.read(200) + except OSError as e: + return [f'{path} cannot be read ({e.strerror or e})'] + r = git(env, '--git-dir', dot, 'rev-parse', '--verify', '--quiet', 'HEAD') + head = r.stdout.decode('ascii', 'replace').strip() if r.returncode == 0 else '' + if not head or text != (head + '\n').encode(): + return [f'{path} is not the one commit (HEAD) a fetch-depth 1 checkout writes: check out with fetch-depth 0 ' + f'or 1'] + return [] def workspace(env): - """(the workspace's real path, None) or (None, why it is refused).""" + """(the workspace's real path, None) or (None, why it is refused). Its real path must be its own path below the + runner's work folder (the parent of RUNNER_WORKSPACE): an earlier job that swapped RUNNER_WORKSPACE or a folder + below it for a symlink would otherwise point the wipe elsewhere, since realpath follows the link on both sides.""" ws = env.get('GITHUB_WORKSPACE') or '' if not ws or not os.path.isabs(ws): return None, f'GITHUB_WORKSPACE {ws!r} is not an absolute path' @@ -152,10 +189,15 @@ def workspace(env): if real == '/': return None, 'GITHUB_WORKSPACE is /' runner = env.get('RUNNER_WORKSPACE') or '' - if runner: - top = os.path.realpath(runner) - if os.path.commonpath([real, top]) != top or real == top: - return None, f'GITHUB_WORKSPACE {real} is not inside RUNNER_WORKSPACE {top}' + if not os.path.isabs(runner): + return None, f'RUNNER_WORKSPACE {runner!r} is not an absolute path' + runner, ws = os.path.normpath(runner), os.path.normpath(ws) + if os.path.commonpath([ws, runner]) != runner or ws == runner: + return None, f'GITHUB_WORKSPACE {ws} is not inside RUNNER_WORKSPACE {runner}' + work = os.path.realpath(os.path.dirname(runner)) + expected = os.path.join(work, os.path.basename(runner), os.path.relpath(ws, runner)) + if real != expected: + return None, f'GITHUB_WORKSPACE {ws} is {real}, not {expected}: a symlink below RUNNER_WORKSPACE' for name in ('HOME', 'RUNNER_TEMP', 'RUNNER_TOOL_CACHE'): other = env.get(name) or '' if other and os.path.isabs(other): @@ -168,19 +210,53 @@ def workspace(env): return real, None -def remove(path): - """Remove one entry without following a symlink; a folder is made writable first (a read-only one too).""" - st = os.lstat(path) - if not stat.S_ISDIR(st.st_mode): - os.unlink(path) - return - if st.st_mode & 0o700 != 0o700: - os.chmod(path, stat.S_IMODE(st.st_mode) | 0o700) - with os.scandir(path) as it: - names = [e.path for e in it] - for p in names: - remove(p) - os.rmdir(path) +DIR_FLAGS = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW + + +def empty(top): + """Remove everything below the folder `top`, depth first with one folder open at a time and by name relative to + it (no recursion, no path longer than a name: any depth works). A symlink is removed as a link, never followed; a + folder is made writable before it is entered (a read-only one too). Raises OSError with the path that failed.""" + fd, down = os.open(top, DIR_FLAGS), [] + try: + while True: + folder = None + for name in sorted(os.listdir(fd)): + try: + st = os.lstat(name, dir_fd=fd) + if stat.S_ISDIR(st.st_mode): + folder = folder or (name, st) + else: + os.unlink(name, dir_fd=fd) + except OSError as e: + e.filename = os.path.join(top, *down, name) + raise + if folder: + name, st = folder + try: + if st.st_mode & 0o700 != 0o700: + os.chmod(name, stat.S_IMODE(st.st_mode) | 0o700, dir_fd=fd) + child = os.open(name, DIR_FLAGS, dir_fd=fd) + except OSError as e: + e.filename = os.path.join(top, *down, name) + raise + os.close(fd) + fd = child + down.append(name) + elif not down: + return + else: # this folder is empty: back up to its parent and remove it + parent = os.open('..', DIR_FLAGS, dir_fd=fd) + os.close(fd) + fd = parent + name = down.pop() + try: + os.rmdir(name, dir_fd=fd) + except OSError as e: + e.filename = os.path.join(top, *down, name) + raise + finally: + os.close(fd) def wipe(env): @@ -192,14 +268,12 @@ def wipe(env): real, why = workspace(env) if why: return f'refused to empty the workspace: {why}' - with os.scandir(real) as it: - names = sorted(e.path for e in it) - for p in names: - try: - remove(p) - except OSError as e: - return f'could not remove {getattr(e, "filename", None) or p} ({e.strerror or e}): the workspace is half empty' - say(f'git-clean: emptied the workspace {real} ({len(names)} entries)') + try: + count = len(os.listdir(real)) + empty(real) + except OSError as e: + return f'could not remove {getattr(e, "filename", None) or real} ({e.strerror or e}): the workspace is half empty' + say(f'git-clean: emptied the workspace {real} ({count} entries)') return None @@ -221,7 +295,9 @@ def check(env, paths, require=False): repo, why = inside(env, p) bad += [why] if why else problems(repo, env, require) for why in bad: - say(f'::error::git-clean: {why}. On a self-hosted runner run fork-ci/actions/git-clean before actions/checkout') + say(f'::error::git-clean: {why}. Left by an earlier job on a self-hosted runner: run fork-ci/actions/git-clean ' + f'before actions/checkout. Written by actions/checkout itself: see "What is refused" in ' + f'fork-ci/actions/git-clean/README.md') return REFUSED if bad else 0 diff --git a/fork-ci/actions/git-clean/git_clean.test.py b/fork-ci/actions/git-clean/git_clean.test.py index cc717565332f4..2bb30ea5c5194 100644 --- a/fork-ci/actions/git-clean/git_clean.test.py +++ b/fork-ci/actions/git-clean/git_clean.test.py @@ -252,6 +252,36 @@ def test_dot_git_file_and_symlink(self): self.assertFalse(os.path.lexists(dot)) self.assertTrue(os.path.isdir(os.path.join(elsewhere, '.git'))) # the target is outside: kept + def test_files_that_change_history_or_objects_are_refused(self): + """grafts change a commit's parents (--no-replace-objects does not turn them off), alternates add another + repository's objects, info/attributes changes how a file merges or diffs. actions/checkout writes none.""" + repo = self.checkout() + git(repo, 'commit', '-q', '--allow-empty', '-m', 'b') + os.makedirs(os.path.join(repo, '.git', 'info'), exist_ok=True) + with open(os.path.join(repo, '.git', 'info', 'grafts'), 'w') as f: + f.write(git(repo, 'rev-parse', 'HEAD') + '\n') + self.assertEqual(git(repo, *G.GIT_SAFE, 'rev-list', '--count', 'HEAD'), '1', 'the planted graft did nothing') + self.assert_refused_then_wiped(repo, 'info/grafts') + for name, text in (('info/attributes', '* merge=union\n'), ('objects/info/alternates', self.outside + '\n'), + ('objects/info/http-alternates', 'https://example.com/objects\n')): + with self.subTest(name=name): + self.rm(self.ws) + os.makedirs(self.ws) + repo = self.checkout() + path = os.path.join(repo, '.git', *name.split('/')) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + f.write(text) + self.assert_refused_then_wiped(repo, name) + + def test_the_refusal_says_what_to_do_either_way(self): # an earlier job's key, or one checkout itself wrote + repo = self.checkout() + self.plant(repo, ('core.sshCommand', 'ssh -i key')) + code, out = self.run_cli('check', '--path', '.', env=self.env('github-hosted')) + self.assertEqual(code, 2, out) + self.assertIn('run fork-ci/actions/git-clean before actions/checkout', out) + self.assertIn('Written by actions/checkout itself', out) + def test_unreadable_config_is_refused(self): repo = self.checkout() with open(os.path.join(repo, '.git', 'config'), 'a') as f: @@ -271,6 +301,7 @@ def test_what_checkout_writes_passes(self): ('filter.lfs.process', 'git-lfs filter-process'), ('filter.lfs.required', 'true'), ('http.https://github.com/.extraheader', 'AUTHORIZATION: basic eDp5'), ('submodule.lib.url', 'https://github.com/owner/lib'), ('submodule.lib.active', 'true'), + ('submodule.ssh.url', 'git@github.com:owner/lib.git'), ('user.name', 'ci'), ('user.email', 'ci@example.com')) git(repo, 'config', '--worktree', 'core.sparseCheckout', 'true') self.assertEqual(G.problems(repo, self.env()), []) @@ -282,14 +313,106 @@ def test_values_are_checked_not_only_keys(self): ('filter.lfs.smudge', 'sh -c x'), ('branch.main.remote', '.'), ('core.bare', 'true'), ('user.name', 'a\nb'), (f'includeIf.gitdir:{self.ws}/.git.path', f'{self.temp}/git-credentials-x.config'), - (f'includeIf.gitdir:{self.ws}/.git.path', f'{self.outside}/git-credentials-{UUID}.config')): + (f'includeIf.gitdir:{self.ws}/.git.path', f'{self.outside}/git-credentials-{UUID}.config'), + (f'includeIf.gitdir:{self.ws}/.git.path', f'{self.temp}/git-credentials-{UUID[:-1]}.config'), + ('include.path', self.creds), ('includeIf.onbranch:main.path', self.creds), + ('http.https://github.com/.extraheader', 'AUTHORIZATION: bearer eDp5'), + ('http.https://github.com/.extraheader', 'AUTHORIZATION: basic eDp5 x'), + ('submodule.lib.url', 'ext::sh -c touch% /tmp/x'), ('submodule.lib.url', 'file:///tmp/x'), + ('submodule.lib.url', 'https://user@example.com/x'), + ('submodule.lib.url', 'git@-oProxyCommand:y')): with self.subTest(key=key, value=value): self.rm(self.ws) os.makedirs(self.ws) repo = self.checkout() + if 'git-credentials' in value and os.path.dirname(value) == self.temp: + self.header_only(value) self.plant(repo, (key, value)) self.assertTrue(G.problems(repo, self.env()), (key, value)) + @property + def creds(self): + return os.path.join(self.temp, f'git-credentials-{UUID}.config') + + def header_only(self, path): + if os.path.exists(path): + os.remove(path) + git(self.d, 'config', '--file', path, 'http.https://github.com/.extraheader', 'AUTHORIZATION: basic eDp5') + + def test_the_token_file_holds_only_the_token_header_value(self): + repo = self.checkout() + os.remove(self.creds) + git(self.d, 'config', '--file', self.creds, 'http.https://github.com/.extraheader', 'AUTHORIZATION: bearer x') + self.assertTrue(G.problems(repo, self.env())) + + def shallow_checkout(self, detached): + """What actions/checkout does with fetch-depth 1, its default: init, fetch one ref with --depth=1, check it out + (detached on pull_request, on a branch on push).""" + src = os.path.join(self.d, 'src') + git(self.d, 'init', '-q', '-b', 'main', src) + for n in ('a', 'b', 'c'): + git(src, 'commit', '-q', '--allow-empty', '-m', n) + repo = self.ws + git(repo, 'init', '-q') + git(repo, 'remote', 'add', 'origin', 'https://github.com/owner/repo') + git(repo, 'fetch', '-q', '--no-tags', '--depth=1', f'file://{src}', '+refs/heads/main:refs/remotes/origin/main') + if detached: + git(repo, 'checkout', '-q', '--detach', 'refs/remotes/origin/main') + else: + git(repo, 'checkout', '-q', '-B', 'main', 'refs/remotes/origin/main') + return repo, src + + def test_a_fetch_depth_1_checkout_passes(self): + for detached in (True, False): + with self.subTest(detached=detached): + self.rm(self.ws) + self.rm(os.path.join(self.d, 'src')) + os.makedirs(self.ws) + repo, _ = self.shallow_checkout(detached) + with open(os.path.join(repo, '.git', 'shallow')) as f: + self.assertEqual(f.read(), git(repo, 'rev-parse', 'HEAD') + '\n') # what checkout leaves + code, out = self.run_cli('check', '--path', '.', '--require', env=self.env('github-hosted')) + self.assertEqual(code, 0, out) + + def test_any_other_shallow_file_is_refused(self): + """shallow cuts history at the commits it names: only the one commit of a depth-1 checkout (HEAD) passes.""" + def deeper(repo, src): # a depth-2 fetch: the boundary is HEAD's parent + git(repo, 'fetch', '-q', '--no-tags', '--depth=2', f'file://{src}', '+refs/heads/main:refs/remotes/origin/main') + + def parent(repo, src): # the history cut below HEAD, in a full clone + self.rm(os.path.join(repo, '.git')) + git(repo, 'init', '-q') + git(repo, 'fetch', '-q', '--no-tags', f'file://{src}', '+refs/heads/main:refs/remotes/origin/main') + git(repo, 'checkout', '-q', '--detach', 'refs/remotes/origin/main') + self.write_shallow(repo, git(src, 'rev-parse', 'HEAD~1') + '\n') + + cases = (('depth 2', deeper), ('parent', parent), + ('two lines', lambda repo, src: self.write_shallow(repo, git(repo, 'rev-parse', 'HEAD') + '\n' + + git(src, 'rev-parse', 'HEAD~1') + '\n')), + ('empty', lambda repo, src: self.write_shallow(repo, '')), + ('no newline', lambda repo, src: self.write_shallow(repo, git(repo, 'rev-parse', 'HEAD'))), + ('symlink', lambda repo, src: self.link_shallow(repo))) + for name, make in cases: + with self.subTest(case=name): + self.rm(self.ws) + self.rm(os.path.join(self.d, 'src')) + os.makedirs(self.ws) + repo, src = self.shallow_checkout(True) + make(repo, src) + code, out = self.run_cli('check', '--path', '.', '--require', env=self.env('github-hosted')) + self.assertEqual(code, 2, out) + self.assertIn('shallow', out) + + def write_shallow(self, repo, text): + with open(os.path.join(repo, '.git', 'shallow'), 'w') as f: + f.write(text) + + def link_shallow(self, repo): + path = os.path.join(repo, '.git', 'shallow') + real = os.path.join(self.d, 'shallow') + os.rename(path, real) + os.symlink(real, path) + def test_no_checkout_yet(self): """GitHub-hosted, before actions/checkout: an empty workspace has nothing to check; --require wants a .git.""" self.assertEqual(self.run_cli('prepare', '--path', '.', env=self.env('github-hosted'))[0], 0) @@ -356,14 +479,67 @@ def test_refuses_a_workspace_it_cannot_trust(self): {'GITHUB_WORKSPACE': self.outside}, # not inside RUNNER_WORKSPACE {'GITHUB_WORKSPACE': self.runner}, # RUNNER_WORKSPACE itself {'GITHUB_WORKSPACE': os.path.join(self.ws, 'nope')}, # missing - {'GITHUB_WORKSPACE': os.path.join(self.d, '_work'), 'RUNNER_WORKSPACE': None}): # holds RUNNER_TEMP + {'GITHUB_WORKSPACE': os.path.join(self.d, '_work'), 'RUNNER_WORKSPACE': None}, + {'RUNNER_WORKSPACE': None}, # self-hosted: the runner always sets it + {'RUNNER_WORKSPACE': 'proj'}, + {'HOME': os.path.join(self.ws, 'sub')}, + {'RUNNER_TEMP': os.path.join(self.ws, 'sub')}, + {'RUNNER_TOOL_CACHE': os.path.join(self.ws, 'sub')}): with self.subTest(bad=bad): code, out = self.run_cli('prepare', '--path', '.', env=self.env(**bad)) self.assertEqual(code, 2, out) self.assertIn('refused', out) + for name in ('HOME', 'RUNNER_TEMP', 'RUNNER_TOOL_CACHE'): + if name in bad: + self.assertIn(f'holds {name}', out) + if bad.get('GITHUB_WORKSPACE') in (self.outside, self.runner): + self.assertIn('is not inside RUNNER_WORKSPACE', out) self.assertEqual(sorted(os.listdir(self.ws)), before) self.assert_outside_kept() + def test_a_symlinked_runner_workspace_is_refused(self): + """An earlier job swapped RUNNER_WORKSPACE (_work/) for a symlink: realpath follows it on both sides, so + only the real path of the runner's work folder tells.""" + elsewhere = os.path.join(self.d, 'elsewhere') + os.makedirs(os.path.join(elsewhere, 'proj')) + with open(os.path.join(elsewhere, 'proj', 'f'), 'w') as f: + f.write('x') + self.rm(self.runner) + os.symlink(elsewhere, self.runner) + code, out = self.run_cli('prepare', '--path', '.') + self.assertEqual(code, 2, out) + self.assertIn('a symlink below RUNNER_WORKSPACE', out) + self.assertEqual(os.listdir(os.path.join(elsewhere, 'proj')), ['f']) + + def test_a_symlink_between_the_runner_workspace_and_the_workspace_is_refused(self): + other = os.path.join(self.runner, 'other', 'proj') + os.makedirs(other) + with open(os.path.join(other, 'f'), 'w') as f: + f.write('x') + os.symlink(os.path.join(self.runner, 'other'), os.path.join(self.runner, 'via')) + code, out = self.run_cli('prepare', '--path', '.', + env=self.env(GITHUB_WORKSPACE=os.path.join(self.runner, 'via', 'proj'))) + self.assertEqual(code, 2, out) + self.assertEqual(os.listdir(other), ['f']) + + def test_a_very_deep_tree_is_removed(self): + """Deeper than Python's recursion limit and longer than PATH_MAX: the next job is not stuck behind it.""" + fd = os.open(self.ws, os.O_RDONLY | os.O_DIRECTORY) + try: + for _ in range(2500): + os.mkdir('d', dir_fd=fd) + down = os.open('d', os.O_RDONLY | os.O_DIRECTORY, dir_fd=fd) + os.close(fd) + fd = down + os.close(os.open('f', os.O_WRONLY | os.O_CREAT, 0o600, dir_fd=fd)) + os.symlink(self.outside, 'out', dir_fd=fd) + finally: + os.close(fd) + code, out = self.run_cli('prepare', '--path', '.') + self.assertEqual(code, 0, out) + self.assertEqual(os.listdir(self.ws), []) + self.assert_outside_kept() + def test_symlinked_workspace_is_refused(self): link = os.path.join(self.runner, 'linked') os.symlink(self.outside, link) @@ -482,6 +658,30 @@ def test_helpers_ignore_global_and_system_config(self): with self.subTest(module=name): self.assertTrue(call('rev-parse', 'HEAD'), f'{name}: git read the global or system config') + def test_helpers_ignore_the_home_attributes_file(self): + """GIT_CONFIG_GLOBAL does not stop git reading ~/.config/git/attributes ($XDG_CONFIG_HOME/git/attributes).""" + home, xdg = os.path.join(self.d, 'home'), os.path.join(self.d, 'xdg') + for d in (os.path.join(home, '.config', 'git'), os.path.join(xdg, 'git')): + os.makedirs(d) + with open(os.path.join(d, 'attributes'), 'w') as f: + f.write('a.txt evil=planted\n') + rest = {k: v for k, v in os.environ.items() if k != 'XDG_CONFIG_HOME'} + for extra in ({'HOME': home}, {'HOME': home, 'XDG_CONFIG_HOME': xdg}): + with unittest.mock.patch.dict(os.environ, {**rest, **extra}, clear=True): + plain = subprocess.run(['git', 'check-attr', 'evil', '--', 'a.txt'], cwd=self.repo, text=True, + capture_output=True, env={**os.environ, 'GIT_CONFIG_GLOBAL': os.devnull, + 'GIT_CONFIG_NOSYSTEM': '1'}).stdout + self.assertIn('planted', plain, 'the planted attributes file is not read: the test plants nothing') + for name, call in self.helpers(): + with self.subTest(module=name, **extra): + out = call('--work-tree', self.repo, 'check-attr', 'evil', '--', 'a.txt') + out = out.decode() if isinstance(out, bytes) else out + self.assertIn('unspecified', out or '', f'{name}: git read the home attributes file') + + def test_no_system_attributes_either(self): # $(prefix)/etc/gitattributes cannot be planted in a test + self.assertEqual(G.GIT_ENV.get('GIT_ATTR_NOSYSTEM'), '1') + self.assertEqual(G.GIT_SAFE[G.GIT_SAFE.index('core.attributesFile=/dev/null') - 1], '-c') + def test_helpers_run_no_planted_fsmonitor(self): for name, call in self.helpers(): if name == 'head_check': @@ -515,6 +715,7 @@ def test_one_source(self): import pr_queue self.assertEqual(pr_queue.GIT_ENV, G.GIT_ENV) self.assertEqual(pr_queue.GIT_SAFE, G.GIT_SAFE) + self.assertEqual(set(pr_queue.UNTRUSTED_FILES), {*G.UNTRUSTED_FILES, 'shallow'}) # git_clean: shallow apart for folder, name in (('job-changed', 'job_changed'), ('docs-scope', 'docs_scope'), ('test-select', 'test_select'), ('pr-queue/tested-base', 'tested_base'), ('weakened-tests', 'weakened_tests')): diff --git a/fork-ci/actions/head-check/README.md b/fork-ci/actions/head-check/README.md index 4565a8430e4a1..c91ba21bb5f03 100644 --- a/fork-ci/actions/head-check/README.md +++ b/fork-ci/actions/head-check/README.md @@ -167,7 +167,7 @@ default branch and posted by an App that only the default branch can use. | `config` | (required) | the queue's JSON config, a path in the repository, read from the base commit | | `required-lenses` | (empty) | lens names, one per line, that must be among the reads; matched as lens keys (case, `r2`) | | `pass-drafts` | `false` | `true` passes a draft without reading its review; add `ready_for_review` to the workflow's triggers | -| `path` | `.` | where the merge commit is checked out | +| `path` | `.` | where the merge commit is checked out, relative to the workspace (an absolute path is refused) | | `pr` | `github.event.pull_request.number` | the pull request | | `repo` | `github.repository` | owner/name | | `token` | `github.token` | reads the pull request's comments and reviews (`pull-requests: read`) | diff --git a/fork-ci/actions/head-check/head_check.py b/fork-ci/actions/head-check/head_check.py index f92330211e9de..992dabf21bfb7 100644 --- a/fork-ci/actions/head-check/head_check.py +++ b/fork-ci/actions/head-check/head_check.py @@ -52,10 +52,10 @@ FIELDS = 'isDraft,headRefOid,comments,reviews' SHA = pr_queue.SHA REFUSED = pr_queue.REFUSED -# Every git call: the queue's safe flags (no hooks, fsmonitor, commit-graph or replace refs) and no attributes file. git -# runs in an empty folder with --git-dir, so the checkout's own .gitattributes (the pull request's) cannot change how +# Every git call: the queue's safe flags (no hooks, fsmonitor, commit-graph, replace refs or attributes file). git runs +# in an empty folder with --git-dir, so the checkout's own .gitattributes (the pull request's) cannot change how # merge-tree merges (merge=union would make a conflict "clean"). -GIT_SAFE = (*pr_queue.GIT_SAFE, '-c', 'core.attributesFile=/dev/null') +GIT_SAFE = pr_queue.GIT_SAFE class Checkout: @@ -81,8 +81,7 @@ def git(self, *a, errors='strict', timeout=None): with tempfile.TemporaryDirectory(prefix='head-check-') as empty: return subprocess.run(['git', *GIT_SAFE, *self.auth, '--git-dir', self.git_dir, *a], cwd=empty, text=True, errors=errors, capture_output=True, timeout=timeout, - env={**os.environ, 'GIT_TERMINAL_PROMPT': '0', 'GIT_ATTR_NOSYSTEM': '1', - **pr_queue.GIT_ENV}) + env={**os.environ, 'GIT_TERMINAL_PROMPT': '0', **pr_queue.GIT_ENV}) def problem(self): """Why the repository cannot be read as git made it, or None: not a git folder, shallow (the base's history is diff --git a/fork-ci/actions/head-check/head_check.test.py b/fork-ci/actions/head-check/head_check.test.py index 329be38b4b1b0..8c8773a133dd9 100644 --- a/fork-ci/actions/head-check/head_check.test.py +++ b/fork-ci/actions/head-check/head_check.test.py @@ -1639,6 +1639,28 @@ def test_the_refusal_does_not_echo_the_pass_drafts_value(self): # a newline in self.assertEqual(r.returncode, 2) self.assertNotIn('forged', r.stdout + r.stderr) + def test_the_git_clean_check_reads_the_path_input(self): + """The first step checks the checkout at `path`, not the workspace: with the good checkout in src/ and a planted + one at the top, path src passes; with the two swapped, it is refused.""" + step = self.action['runs']['steps'][0] + self.assertIn('git_clean.py', step['run']) + ws = tempfile.mkdtemp(prefix='head-check-ws-') + self.addCleanup(shutil.rmtree, ws, True) + env = {k: {'${{ inputs.path }}': 'src'}[v] for k, v in step['env'].items()} + for good, bad in (('src', '.'), ('.', 'src')): + with self.subTest(good=good): + for p in ('.git', 'src'): + shutil.rmtree(os.path.join(ws, p), True) + for p in (good, bad): + subprocess.run(['git', 'init', '-q', os.path.join(ws, p)], check=True, capture_output=True, + env={**os.environ, 'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'}) + with open(os.path.join(ws, bad, '.git', 'config'), 'a') as f: + f.write('[core]\n\tfsmonitor = true\n') + r = subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', step['run']], cwd=ws, + env={'PATH': os.environ['PATH'], 'GITHUB_ACTION_PATH': HERE, 'GITHUB_WORKSPACE': ws, + **env}, capture_output=True, text=True) + self.assertEqual(r.returncode, 0 if good == 'src' else 2, r.stdout + r.stderr) + def test_the_defaults_are_the_pull_request_events(self): inputs = self.action['inputs'] self.assertEqual(inputs['pr']['default'], '${{ github.event.pull_request.number }}') diff --git a/fork-ci/actions/job-changed/job_changed.py b/fork-ci/actions/job-changed/job_changed.py index 5a4dbc80d2376..89cd7d39d98ab 100644 --- a/fork-ci/actions/job-changed/job_changed.py +++ b/fork-ci/actions/job-changed/job_changed.py @@ -30,12 +30,12 @@ NO_YAML = 'PyYAML is not installed on this runner (install python3-yaml), the change cannot be read' -# Every git call: no global or system config (on a self-hosted runner an earlier job may have written ~/.gitconfig), and -# no hooks, fsmonitor command, commit-graph or replace refs from the repository. The same as +# Every git call: no global or system config or attributes (on a self-hosted runner an earlier job may have written +# ~/.gitconfig), and no hooks, fsmonitor command, commit-graph or replace refs from the repository. The same as # ../git-clean/git_clean.py's (git_clean.test.py pins them); the action's first step refused any other .git config. -GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'} +GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} GIT_SAFE = ('-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.commitGraph=false', - '--no-replace-objects') + '-c', 'core.attributesFile=/dev/null', '--no-replace-objects') def git(*args): diff --git a/fork-ci/actions/pr-queue/README.md b/fork-ci/actions/pr-queue/README.md index 3907933611296..494b77cd0eaf4 100644 --- a/fork-ci/actions/pr-queue/README.md +++ b/fork-ci/actions/pr-queue/README.md @@ -538,7 +538,8 @@ checkout of its config, in a job with the read permissions above: With `update_when_base_was_red`, the job of every required check runs `tested-base` right after its checkout of the merge commit (`path` is the -checkout's path, default `.`): +checkout's path relative to the workspace, default `.`; an absolute path or a +`.git` that fork-ci git-clean refuses writes no note): ```yaml - uses: actions/checkout@ diff --git a/fork-ci/actions/pr-queue/pr_queue.py b/fork-ci/actions/pr-queue/pr_queue.py index e0f26efe6526e..41bb6128fe347 100755 --- a/fork-ci/actions/pr-queue/pr_queue.py +++ b/fork-ci/actions/pr-queue/pr_queue.py @@ -73,10 +73,11 @@ # Every git call, whatever the repository's files say: no hooks, no fsmonitor command, no replace refs, and no # commit-graph (a planted objects/info/commit-graph(s) could change the parents git reports; git rebuilds nothing from it). GIT_SAFE = ('-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.commitGraph=false', - '--no-replace-objects') -# ... and no global or system config: on a self-hosted runner an earlier job may have written ~/.gitconfig. The same + '-c', 'core.attributesFile=/dev/null', '--no-replace-objects') +# ... and no global or system config or attributes: on a self-hosted runner an earlier job may have written +# ~/.gitconfig or ~/.config/git/attributes. The same # two as ../git-clean/git_clean.py's (git_clean.test.py pins GIT_SAFE and GIT_ENV to its). -GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'} +GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} # The cache's whole config, as the queue writes it: a cache with any other config (a planted `url.*.insteadOf`, helper # or hook path) is rebuilt, never run. CACHE_CONFIG = '[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = true\n' diff --git a/fork-ci/actions/pr-queue/pr_queue.test.py b/fork-ci/actions/pr-queue/pr_queue.test.py index cae753b903a69..fdd386160e492 100644 --- a/fork-ci/actions/pr-queue/pr_queue.test.py +++ b/fork-ci/actions/pr-queue/pr_queue.test.py @@ -2670,12 +2670,13 @@ def git_call(self, read=None, write=None): def test_git_gets_the_login_from_gh_only_and_never_prompts(self): q, argv, env = self.git_call('run-token', 'pat-token') - self.assertEqual(argv[:13], ['git', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', - '-c', 'core.commitGraph=false', '--no-replace-objects', - '-c', 'credential.https://github.com.helper=', + self.assertEqual(argv[:15], ['git', '-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', + '-c', 'core.commitGraph=false', '-c', 'core.attributesFile=/dev/null', + '--no-replace-objects', '-c', 'credential.https://github.com.helper=', '-c', 'credential.https://github.com.helper=!gh auth git-credential', '--git-dir']) - self.assertEqual(argv[13:], [q.cache, 'fetch', 'x']) + self.assertEqual(argv[15:], [q.cache, 'fetch', 'x']) self.assertEqual(env['GIT_TERMINAL_PROMPT'], '0') + self.assertEqual(env['GIT_ATTR_NOSYSTEM'], '1') def test_git_fetches_with_the_read_token_only(self): # gh auth git-credential answers with GH_TOKEN q, argv, env = self.git_call('run-token', 'pat-token') diff --git a/fork-ci/actions/pr-queue/tested-base/tested_base.py b/fork-ci/actions/pr-queue/tested-base/tested_base.py index 46fc97d6067ee..71782a7f6c8a7 100644 --- a/fork-ci/actions/pr-queue/tested-base/tested_base.py +++ b/fork-ci/actions/pr-queue/tested-base/tested_base.py @@ -18,12 +18,12 @@ SHA = re.compile(r'[0-9a-f]{40}') -# Every git call: no global or system config (on a self-hosted runner an earlier job may have written ~/.gitconfig), and -# no hooks, fsmonitor command, commit-graph or replace refs from the repository. The same as +# Every git call: no global or system config or attributes (on a self-hosted runner an earlier job may have written +# ~/.gitconfig), and no hooks, fsmonitor command, commit-graph or replace refs from the repository. The same as # ../../git-clean/git_clean.py's (git_clean.test.py pins them); the action's first step refused any other .git config. -GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'} +GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} GIT_SAFE = ('-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.commitGraph=false', - '--no-replace-objects') + '-c', 'core.attributesFile=/dev/null', '--no-replace-objects') def git(path, *a): diff --git a/fork-ci/actions/weakened-tests/README.md b/fork-ci/actions/weakened-tests/README.md index 2f9322bb31c01..a0a36bb3f6adc 100644 --- a/fork-ci/actions/weakened-tests/README.md +++ b/fork-ci/actions/weakened-tests/README.md @@ -30,7 +30,7 @@ Listen to `edited` too (the example does), so a body edit that adds a "why" upda | ---------------- | --------------------------------------------- | ------------------------------------------------------------------ | | `config` | `.github/weakened-tests.json` | the project's config, read at the base commit; optional file | | `base`, `head` | the event's `pull_request.base/head.sha` | the change is `merge-base(base, head)..head` | -| `path` | `.` | where the repository is checked out | +| `path` | `.` | where the repository is checked out, relative to the workspace | | `pr`, `repo` | the event's pull request, `github.repository` | where the comment goes | | `token` | `github.token` | reads the pull request, posts and edits its comments | | `comment` | `true` | `false`: only the job summary and the outputs | @@ -38,7 +38,7 @@ Listen to `edited` too (the example does), so a body edit that adds a "why" upda | Output | Value | | ------------- | ------------------------------------------------------------------------------------------- | -| `status` | `ok` or `could-not-read` | +| `status` | `ok`, `could-not-read`, or `refused` (git-clean refused the checkout: nothing was checked) | | `count` | items listed; empty when the change could not be read | | `missing_why` | listed items the body gives no why for; empty when the change or the body could not be read | @@ -113,6 +113,10 @@ for js, jsx, ts, tsx, mjs, cjs, mts, cts; `it(` and `test(` with a string name). - **Could not read** (a commit missing from the checkout, no merge base, a git error, a bad config, test files over `max_bytes`, an unexpected error) is said in the comment and the summary, never shown as an empty list. A failure to read the body, list the comments or post is a warning; the step still passes. +- **Refused:** the first thing the step runs is fork-ci git-clean's check of the checkout (`../git-clean/README.md`). + When it refuses (a `.git` an earlier job left, an absolute `path`), no git runs, `status` is `refused`, and an + existing report comment is replaced by a note that this head was not checked (no comment is posted when there is + none). The step still passes. ## Why a pull request cannot loosen it diff --git a/fork-ci/actions/weakened-tests/action.yml b/fork-ci/actions/weakened-tests/action.yml index 2bb985b3795aa..6289534a2e847 100644 --- a/fork-ci/actions/weakened-tests/action.yml +++ b/fork-ci/actions/weakened-tests/action.yml @@ -43,7 +43,10 @@ inputs: default: github-actions[bot] outputs: status: - description: ok, or could-not-read (empty when the step did not run). + description: >- + ok; could-not-read (the change could not be read: README "Could not read"); refused (git-clean refused the + checkout's git config: nothing was checked, and an earlier report comment says so); empty when the step did not + run. value: ${{ steps.report.outputs.status }} count: description: How many items are listed (empty when the change could not be read). @@ -56,7 +59,8 @@ runs: steps: # Only code from this action's folder (the caller's pinned SHA) runs; nothing from the checkout. Before any git # call, the checkout's .git config must be one actions/checkout writes (fork-ci git-clean): when it is not (one an - # earlier job left), no git runs and there is no report; report only, so the step still passes. + # earlier job left), no git runs, status is refused and an earlier report comment is replaced; report only, so the + # step still passes. - id: report shell: bash env: @@ -70,12 +74,12 @@ runs: COMMENT: ${{ inputs.comment }} COMMENT_AUTHOR: ${{ inputs.comment-author }} run: | - if ! python3 -I "$GITHUB_ACTION_PATH/../git-clean/git_clean.py" check --path "$CHECKOUT_PATH" --require; then - echo "::warning::weakened-tests: no report, git-clean refused the checkout's git config (above)" - exit 0 - fi args=(--config "$CONFIG" --repo "$REPO" --pr "$PR" --base "$BASE" --head "$HEAD" --path "$CHECKOUT_PATH" --comment-author "$COMMENT_AUTHOR") [ "$COMMENT" = true ] || args+=(--no-comment) + if ! python3 -I "$GITHUB_ACTION_PATH/../git-clean/git_clean.py" check --path "$CHECKOUT_PATH" --require; then + echo "::warning::weakened-tests: no report, git-clean refused the checkout's git config (above)" + args+=(--refused) + fi python3 -I "$GITHUB_ACTION_PATH/weakened_tests.py" "${args[@]}" || echo "::warning::weakened-tests: the report step failed (exit $?); report only, the pull request is not failed" diff --git a/fork-ci/actions/weakened-tests/weakened_tests.py b/fork-ci/actions/weakened-tests/weakened_tests.py index 93460b7dad0c6..e81dd70f0a0f0 100644 --- a/fork-ci/actions/weakened-tests/weakened_tests.py +++ b/fork-ci/actions/weakened-tests/weakened_tests.py @@ -26,9 +26,11 @@ Every error, including one this script did not foresee, is a ::warning:: and exit 0. python3 -I weakened_tests.py --config .github/weakened-tests.json --repo OWNER/NAME --pr N --base SHA --head SHA - [--path DIR] [--comment-author LOGIN] [--no-comment] -Writes status=ok|could-not-read, count= and missing_why= to $GITHUB_OUTPUT (count and -missing_why are empty when the change could not be read; missing_why is empty when the PR body could not be read). + [--path DIR] [--comment-author LOGIN] [--no-comment] [--refused] +Writes status=ok|could-not-read|refused, count= and missing_why= to $GITHUB_OUTPUT (count +and missing_why are empty when the change could not be read; missing_why is empty when the PR body could not be read). +--refused: git-clean refused the checkout, so no git runs; status is refused, and a report comment of an earlier head +is replaced by a note (none is posted when there is none). """ import argparse, collections, copy, json, os, re, subprocess, sys @@ -88,11 +90,11 @@ class Unclear(Exception): """The change or the config could not be read: the report says so instead of listing nothing.""" -# No global or system config, no hooks, fsmonitor command, commit-graph or replace refs from the repository: the same -# two as ../git-clean/git_clean.py's (git_clean.test.py pins them to its; this file stays self-contained). -GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1'} +# No global or system config or attributes, no hooks, fsmonitor command, commit-graph or replace refs from the +# repository: the same two as ../git-clean/git_clean.py's (git_clean.test.py pins them to its; this file stays self-contained). +GIT_ENV = {'GIT_CONFIG_GLOBAL': os.devnull, 'GIT_CONFIG_NOSYSTEM': '1', 'GIT_ATTR_NOSYSTEM': '1'} GIT_SAFE = ('-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.commitGraph=false', - '--no-replace-objects') + '-c', 'core.attributesFile=/dev/null', '--no-replace-objects') class Git: @@ -320,8 +322,15 @@ def code(text): return f'`{text.strip() or " "}`' -def render(items, reason, why, merge_base, head): +REFUSED_TEXT = ('**No report for head {head}:** git-clean refused the checkout\'s git config (the job log says why), ' + 'so nothing was checked. An earlier report here was for another head.') + + +def render(items, reason, why, merge_base, head, refused=False): lines = [MARKER, '### Existing tests this pull request deletes, skips or weakens', ''] + if refused: + lines += [REFUSED_TEXT.format(head=code(head[:12])), '', 'Report only: this never fails the pull request.'] + return '\n'.join(lines) + '\n' if items is None: lines += [f'**Could not read the change:** {code(reason)}. This is not an empty list: nothing was checked.', '', 'Report only: this never fails the pull request.'] @@ -400,11 +409,14 @@ def cli(argv=None, env=None, gh=None, out=print): p.add_argument('--path', default='.') p.add_argument('--comment-author', default='github-actions[bot]') p.add_argument('--no-comment', action='store_true') + p.add_argument('--refused', action='store_true') try: a = p.parse_args(argv) except SystemExit: out('::warning::weakened-tests: bad arguments; nothing was checked (this never fails the pull request)') return 0 + if a.refused: + return refused(a, env, gh, out) items, reason, mb = None, '', '' try: items, mb = find(a.base, a.head, a.config, Git(a.path)) @@ -441,5 +453,21 @@ def cli(argv=None, env=None, gh=None, out=print): return 0 +def refused(a, env, gh, out): + """git-clean refused the checkout: no git, status refused, and an earlier head's report comment replaced.""" + text = render(None, '', [], '', a.head, refused=True) + try: + append(env.get('GITHUB_STEP_SUMMARY'), text) + append(env.get('GITHUB_OUTPUT'), 'status=refused\ncount=\nmissing_why=\n') + except OSError as e: + out(f'::warning::weakened-tests: could not write the outputs: {escape(e)}') + if not a.no_comment: + try: + out(f'weakened-tests: comment {publish(gh, a.repo, a.pr, text, a.comment_author, True)}') + except Exception as e: # noqa: BLE001 + out(f'::warning::weakened-tests: {escape(e)}') + return 0 + + if __name__ == '__main__': sys.exit(cli()) diff --git a/fork-ci/actions/weakened-tests/weakened_tests.test.py b/fork-ci/actions/weakened-tests/weakened_tests.test.py index c0dd60f3b7c5b..8ea6b4707c394 100644 --- a/fork-ci/actions/weakened-tests/weakened_tests.test.py +++ b/fork-ci/actions/weakened-tests/weakened_tests.test.py @@ -3,7 +3,7 @@ a fake `gh` that records each call. Run: python3 fork-ci/actions/weakened-tests/weakened_tests.test.py """ -import json, os, re, shutil, stat, subprocess, sys, tempfile, unittest +import json, os, re, shutil, stat, subprocess, sys, tempfile, unittest, unittest.mock import yaml @@ -471,6 +471,36 @@ def test_an_unforeseen_error_is_could_not_read(self): self.assertIn('Could not read', gh.ours()[0]['body']) +class Refused(unittest.TestCase): + """git-clean refused the checkout (the action passes --refused): no git runs, status is refused, and the report + comment of an earlier head is replaced, so it is not read as this head's.""" + + def run_refused(self, gh): + r = Repo(self) + c = Cli(self, r) + with unittest.mock.patch.object(wt, 'Git', side_effect=AssertionError('git ran')): + out = c.run(r.base, gh, extra=('--refused',)) + return c, out + + def test_status_refused_and_the_old_report_is_replaced(self): + gh = FakeGh(comments=[{'id': 3, 'login': BOT, 'body': MARK + '\nold list of an earlier head'}]) + c, out = self.run_refused(gh) + self.assertEqual((out['status'], out['count'], out['missing_why']), ('refused', '', '')) + self.assertEqual(gh.writes(), [('PATCH', 'repos/o/r/issues/comments/3')]) + self.assertIn('git-clean refused', gh.ours()[0]['body']) + self.assertNotIn('old list', gh.ours()[0]['body']) + self.assertIn('git-clean refused', c.summary) + + def test_no_comment_yet_posts_none_and_again_edits_nothing(self): + gh = FakeGh() + self.run_refused(gh) + self.assertEqual(gh.writes(), []) # nothing stale to replace + gh = FakeGh(comments=[{'id': 3, 'login': BOT, 'body': MARK + '\nold'}]) + self.run_refused(gh) + self.run_refused(gh) # runs again: the same text, so no second edit + self.assertEqual(gh.writes(), [('PATCH', 'repos/o/r/issues/comments/3')]) + + class Comment(unittest.TestCase): def weakened(self, r): r.branch() @@ -668,6 +698,11 @@ def test_a_git_config_checkout_never_writes_means_no_report_and_no_git(self): self.addCleanup(shutil.rmtree, files, True) marker = os.path.join(files, 'ran') r.git('config', 'core.fsmonitor', f'touch {marker}; false') + gh = os.path.join(files, 'gh') # the comment list holds the report of an earlier head + with open(gh, 'w') as f: + f.write('#!/bin/sh\necho "$*" >> "$GH_LOG"\ncase "$*" in *--paginate*) printf \'%s\\n\' ' + f"'{json.dumps({'id': 3, 'login': BOT, 'body': MARK + ' old'})}';; *PATCH*) cat >> \"$GH_LOG\";; esac\n") + os.chmod(gh, 0o755) given = {'config': CONFIG, 'repo': 'o/r', 'pr': '7', 'base': r.base, 'head': head, 'path': '.', 'token': 'x', 'comment': 'true', 'comment-author': BOT} values = {k: given[re.fullmatch(r'\$\{\{ inputs\.([\w-]+) \}\}', v)[1]] for k, v in step['env'].items()} @@ -676,13 +711,18 @@ def test_a_git_config_checkout_never_writes_means_no_report_and_no_git(self): p = subprocess.run(['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', step['run']], text=True, capture_output=True, cwd=r.d, env={**os.environ, **values, 'GITHUB_OUTPUT': out, 'GITHUB_ACTION_PATH': HERE, - 'GITHUB_WORKSPACE': r.d, 'PATH': files + os.pathsep + os.environ['PATH']}) + 'GITHUB_WORKSPACE': r.d, 'GH_LOG': os.path.join(files, 'gh.log'), + 'PATH': files + os.pathsep + os.environ['PATH']}) self.assertEqual(p.returncode, 0, p.stdout + p.stderr) self.assertIn('::error::git-clean: ', p.stdout) self.assertIn('::warning::weakened-tests: no report', p.stdout) self.assertFalse(os.path.exists(marker)) with open(out) as f: - self.assertEqual(f.read(), '') + self.assertEqual(f.read(), 'status=refused\ncount=\nmissing_why=\n') + with open(os.path.join(files, 'gh.log')) as f: + log = f.read() + self.assertIn('PATCH repos/o/r/issues/comments/3', log) # the earlier head's report is replaced + self.assertIn('git-clean refused', log) def test_the_example_caller_runs_it_one_at_a_time_with_full_history(self): with open(os.path.join(HERE, 'weakened-tests-caller.example.yml')) as f: