From 3249525c4fc4e569046eef5f2c1c20872ee66d8f Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:20:50 +0000 Subject: [PATCH] cleaner: agent-scratch rule with a keep-first step (#489) PR #512 (reviewed at 534a5c18: the rule, the lens fix round, the README tables and the round-2 fixes) squashed and replayed onto main after #511 (compress-cold). Both rules kept whole: - engine delete_all: compress items stay out of the journal (rules.UNJOURNALED); session items keep their keep-first copy, the re-read before the first unlink, and the one-journal-line rule for a delete an earlier run began. - rules: RULES, recheck and delete name both kinds. - config/report/tests: both sections, both report classes and both test suites; the scratch red check names the merged journal line. - New, from the two together: compress-cold never takes the agent-scratch root (a folder another rule deletes from) or its keep-first archive (one of the cleaner's own files) while that rule is on, with a test and two red checks. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 1 + fork-kit/cleaner/README.md | 218 ++++- fork-kit/cleaner/lib/cleaner/compare.py | 4 +- fork-kit/cleaner/lib/cleaner/config.py | 158 +++- fork-kit/cleaner/lib/cleaner/engine.py | 82 +- fork-kit/cleaner/lib/cleaner/facts.py | 194 ++++- fork-kit/cleaner/lib/cleaner/guard.py | 10 +- fork-kit/cleaner/lib/cleaner/report.py | 13 +- fork-kit/cleaner/lib/cleaner/rules.py | 599 +++++++++++++- fork-kit/cleaner/tests/cleaner_fixture.py | 28 +- fork-kit/cleaner/tests/compress.test.py | 24 + fork-kit/cleaner/tests/engine.test.py | 16 +- fork-kit/cleaner/tests/guard.test.py | 6 +- fork-kit/cleaner/tests/report.test.py | 8 +- fork-kit/cleaner/tests/scratch.test.py | 965 ++++++++++++++++++++++ fork-kit/cleaner/tests/worktrees.test.py | 4 +- 16 files changed, 2211 insertions(+), 119 deletions(-) create mode 100644 fork-kit/cleaner/tests/scratch.test.py diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 94d5d2125695c..5bd53a89ecba3 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -69,6 +69,7 @@ fork-kit/cleaner/tests/engine.test.py fork-kit/cleaner/tests/guard.test.py fork-kit/cleaner/tests/install.test.py fork-kit/cleaner/tests/report.test.py +fork-kit/cleaner/tests/scratch.test.py fork-kit/cleaner/tests/worktrees.test.py ~8s fork-kit/common/tests/free_contract.test.py ~5s fork-kit/common/tests/machine_file.test.py diff --git a/fork-kit/cleaner/README.md b/fork-kit/cleaner/README.md index d18ed59cd1813..70f090e80426f 100644 --- a/fork-kit/cleaner/README.md +++ b/fork-kit/cleaner/README.md @@ -1,13 +1,14 @@ # cleaner: free disk space by rules, and never delete what could be needed -`fork-cleaner` removes what a host no longer needs: finished git worktrees, unused anonymous Docker volumes and old -temp folders. Which folders, which repositories and how many days come from the machine file. What must never go +`fork-cleaner` removes what a host no longer needs: finished git worktrees, unused anonymous Docker volumes, old +temp folders, and the scratch folders of agent sessions that are over (after it has copied out the small reports +in them worth keeping). Which folders, which repositories and how many days come from the machine file. What must never go is in the code and cannot be switched off. It lives in `fork-kit` because it runs on a machine (not in CI and not inside T3), reads its facts from the machine file like every kit, and ships with the kits in `ci-release`. `fork-ci` holds only what CI itself runs. -The core, three rules, an [installer](#install) and one [daily timer](#the-daily-timer). While the tools it is +The core, four rules, an [installer](#install) and one [daily timer](#the-daily-timer). While the tools it is to replace still delete on a host, the timer runs in compare mode: it plans, reports and compares what it would delete with what they would, and deletes nothing. @@ -16,7 +17,7 @@ delete with what they would, and deletes nothing. | config | `lib/cleaner/config.py` | reads the `cleaner` section of the machine file | | facts | `lib/cleaner/facts.py` | reads the process list, systemd units, git, GitHub and Docker | | guard | `lib/cleaner/guard.py` | the hard refusals: asked about every path, twice | -| rules | `lib/cleaner/rules.py` | `worktrees`, `docker`, `tmp`: what each wants gone, and why | +| rules | `lib/cleaner/rules.py` | `worktrees`, `docker`, `tmp`, `agent-scratch`: what and why | | compress | `lib/cleaner/compress.py` | `compress-cold`: cold files replaced by a checked zstd copy | | `plan`,`run` | `lib/cleaner/engine.py` | the two commands, the lock and the log | | `report` | `lib/cleaner/report.py` | one line: GB per class, and what runs freed | @@ -52,8 +53,12 @@ and clones of one repository makes N + 1 reads. Each item's last check also runs time, never through a link: the entry must still be the folder (same device and inode) the last check judged, and every file and folder removed under it must be one the last check walked. A link or another folder swapped in after the last check is refused with `FAILED ... is not what the last check saw ... nothing removed` (exit -1); something moved into it since stops the delete there (`not there at the last check`, half deleted). A -worktree is checked the same way just before `git worktree remove` runs. +1); something moved into it since stops the delete there (`not there at the last check`, half deleted). An +agent session's folder is also stopped by anything whose inode time is after the check right before its delete +(`changed after the last check`, half deleted): ext4 hands a freed inode number to the next new file at once, so +the inode number alone does not show a file made during the delete. A file with several hard links is let +through (deleting one name changes the others' inode time). A worktree is checked the same way just before +`git worktree remove` runs. It refuses to run as root (exit 2): `~` would then be root's home, and the must-keep paths would miss the user's own agent folders. @@ -103,6 +108,13 @@ that cannot be written stops the run before that delete. A journal that cannot b file, stops `run` before any delete; `plan` says so, lists every planned path as `kept ... could not read` instead of `would delete`, and exits 1. +One exception: an [`agent-scratch`](#agent-scratch) session folder is judged again by the next run, as #466 +asked. When every check still passes for what is left (the session is still gone, no lock, nothing in use, no +git work only there) and no file in it changed after the delete began (its journal time), the run finishes the +delete (`finishing the delete an earlier run began at ...`), keeping its journal line until it is done. Otherwise +it stays `half deleted`, with the reason in front: `kept (agent-scratch: half deleted: since ...: session +alive: ...)`. + ## The report `fork-cleaner report` prints one line for the daily report of this host, for example: @@ -119,7 +131,8 @@ not be read) (180.2 GB of 250.0 GB used); freed since the last report at 2026-01 - **The classes** are `CLASSES` in `config.py`, the one list, and each name says what it measures. A rule's class is what that rule looks at: `worktrees` (the worktrees and clones of the listed repositories), `temp` (the temp - entries a pattern takes) and `all docker volumes` (every volume, named ones too: Docker measures them all, and + entries a pattern takes), `agent scratch` (every session folder of the agent-scratch root, idle or not; `off` + while the rule is off) and `all docker volumes` (every volume, named ones too: Docker measures them all, and the rule takes only unused anonymous ones). A must-keep class is its group of must-keep paths, whole: `agent tool homes` (`AGENT_HOMES`: the agent tools' home folders, with transcripts, sessions, settings and caches) and `T3 data` (`T3_DATA`). `build caches` has no rule yet. Every rule of `rules.RULES` (the list `plan` @@ -203,7 +216,7 @@ off this list. itself (a folder without read permission) cannot be read in full, what was asked about stays, the line says `could not read`, and the command ends with exit code 1. -## The three rules +## The four rules ### `worktrees` @@ -265,7 +278,116 @@ default list is empty: with no pattern the rule does nothing. "Changed" is the n entry and of everything in it, so a folder copied today with old dates is new. Never taken, whatever the patterns say: hidden entries, `claude-*`, `tmux-*`, `ssh-*`, `systemd-private-*`, -`snap-private-tmp`, `snap.*` and `lost+found`. Agent scratch folders get a rule of their own later. +`snap-private-tmp`, `snap.*` and `lost+found`. Agent scratch folders have a rule of their own: +[`agent-scratch`](#agent-scratch). + +### `agent-scratch` + +Off unless the machine file says `"enabled": true`. Agent sessions leave a scratch folder per session in the temp +folder (Claude Code: `/tmp/claude-///` with `scratchpad/`, `tasks/` and +`images/`), and nothing removes them. The rule looks at the folders two levels below `root` (default +`/tmp/claude-{uid}`, `{uid}` being the user's id) whose name is a session id (a uuid); the project folders +themselves never go, and a project folder that is a link is kept and named. The root must be one user's own +folder: never a shared temp folder itself (`/tmp`, `/var/tmp`, `/dev/shm`, `/run/user`, the user's own runtime +folder `/run/user/` or `$XDG_RUNTIME_DIR`, `$TMPDIR`), never `/`, home or above it, never a folder that is, +holds or is inside a `transcripts` folder (by its path or its real path: agent logs are never deleted), and when +it is there, a real folder (not a link) of the cleaner's user; anything else stops the run as a machine file +error. A session folder goes only when all of this holds, the hard refusals of +[What it never deletes](#what-it-never-deletes) included: + +| Kept as | When | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| (not listed) | anything in it changed less than `days` ago (3 or more): the newest content or inode time of all of it | +| `lock file` | a lock-named file is directly in it (`*.lock`, `lock`, `*.lck`); deeper, a clone's `yarn.lock` or `Cargo.lock` is ordinary content (a lock a process _holds_ anywhere in it is `in use`) | +| `transcript changed under 3 days ago` | the session's transcript (`.jsonl` or its `/` folder in a folder `transcripts` names) changed within `days` | +| `session alive`, `in use` | the session may still be alive: see [What counts as alive](#what-counts-as-alive) | +| `could not read` | the process list, a process of the user (see below), a transcript folder or a folder in it cannot be read in full, or a `transcripts` pattern matches no folder | +| `git: unpushed commits`, ... | a clone in it holds uncommitted, untracked, stashed or unpushed work: kept, and the clone is named | +| `report too large to keep first` | a file a `keepFirst.patterns` pattern takes is over `maxFileBytes`: it stays until a person keeps it or it is gone (nothing is deleted before it is saved) | +| `too much to keep first` | the files to keep first add up to more than `maxSessionBytes` | +| `credential name`, ... | every refusal of the list above | + +Transcripts are read by their times only, never their content, and are never changed (they are must-keep). Where +they are comes from the machine file (`transcripts`, default `~/.claude*/projects/*`: one folder per project); +each match is resolved to its real path, so a config folder that is a link (or `CLAUDE_CONFIG_DIR` named there) +works. A pattern that matches no folder at all is `could not read` for every session, never "no transcript": a +transcript folder that moved must not make every session look gone. + +##### What counts as alive + +Asked last in the plan, again by the last check before each delete, and once more after keep first, right before +the first unlink (the copy can take a while). A session folder is kept when any of these holds: + +1. `in use` (the plan and the last check) or `session alive` (after keep first): a process of any user works in + the folder or below it (its working folder or root), has a file or folder in it open (`/proc//fd`, a + deleted file included; a running session holds its `tasks/` folder open), maps a file in it, listens on a + socket in it, holds a lock in it, or names its path on its command line. +2. `session alive`: a process names the id as a whole word, on its command line (`--session-id`, `--resume `) + or in its environment (`CLAUDE_CODE_SESSION_ID`, which every tool and MCP server of a session inherits). Of other + users' processes only the command line is visible. +3. `could not read`: the process list cannot be read, or a process of the cleaner's own user hides its open files, + working folder or environment (a process that is not dumpable, such as the Codex CLI) and has no readable + ancestor to be judged by. A hidden process is judged by its nearest ancestor (`/proc//stat` shows the + parent even of a hidden process) that is the user's own and can be read in full: what that ancestor works in, + holds open and names decides, under 1, 2 and 4 like any process (a Codex started by an agent's tool is that + agent's session's; one started by the T3 server is no session's). Hidden ancestors are walked past; the walk + stops, with no ancestor, at pid 1, the user's service manager (`systemd --user`), a process of another user or + one that has ended. The login session's helpers that hide on purpose and hold no session (`systemd`, + `(sd-pam)`, `gpg-agent`, `ssh-agent`, `scdaemon`, `keyboxd`, `gnome-keyring-d`, by `/proc//comm`) need no + ancestor. Their name excuses only what cannot be read: what can be read of such a process (its command line, + such as a `--homedir` in a session; a socket in `/proc/net/unix`) still counts under 1 and 2. +4. `session alive`: after `/clear` (or a resume under a new id) the same process works on under another id. So a + process of the user that works in or holds open another session folder of the _same project_ without naming + that session's id, and that started before this session last changed (its folder or its transcript; a start + time that cannot be read counts as before), keeps this session. +5. `transcript changed under ... ago`: as above. + +Not covered: an interactive `claude` (no id on its command line) after `/clear`, when it holds nothing in the old +folder and its tools carry only the new id. (A T3-started session is covered: its process names its first id on +its command line for as long as it lives, so 2 keeps the old folder.) Also not covered: a hidden process whose +readable ancestor works elsewhere, but which itself works in a session folder (it cannot be seen). Its old folder goes once it and its transcript are `days` old; what is worth +keeping in it is kept first. + +#### Keep first + +Before a session folder is deleted, the files in it worth keeping for analysis are copied out (#466: "before raw +data is deleted, check whether anything in it is worth keeping; save that first; only then delete"). What is +worth keeping is set in the machine file (`keepFirst.patterns`, relative to the session folder; `*` within one +part). The default takes the small text reports an agent writes in its scratchpad, and not the clones, +downloads, media and build output deeper in it: + +``` +scratchpad/*.md scratchpad/*.json scratchpad/*.txt scratchpad/*.tsv scratchpad/*.csv scratchpad/*/*.md +``` + +The list cannot be empty (keep first cannot be turned off; `[]` is a machine file error). A report over +`maxFileBytes` (256 KB) is not copied and keeps the whole session (`report too large to keep first`). Links are +never followed, and nothing inside a git folder or a clone is taken. Nothing else is read or analysed. + +The full output of the session's background commands is kept too (`keepFirst.outputs`, default +`tasks/*.output`; a transcript holds only what the agent read of it), up to the same `maxFileBytes` each. An output +that is a link is followed only to a regular file inside the session folder; a link out (a sub-agent's output +links to its transcript, which stays anyway) is never followed and is listed in the manifest's `linksOut`. A +larger output is not copied and does not hold the session: the manifest's `notCopied` names its path, size and +sha256, so the record of what was there stays. + +Every file is read by descriptor, one folder at a time from the session folder and never through a link, and only +when it is the file the last check walked; keep first refuses (and the folder stays) when what it finds is not what +the last check saw. The copy goes to `///` (default archive +`~/.local/state/fork-cleaner/kept`), with a `manifest.json`: the source folder, the project, the session, the time, +and each file's path, size and sha256. New archive folders are made with mode 0700, each synced into its parent, +and every folder from the archive down must be a real folder of the cleaner's user (a link there fails keep +first). The copy is written as `..partial`, every file and folder synced, read back and checked +against the sha256 of what was read from the session, renamed into place (the parent synced), and read back once +more. A `..partial` already there is the cleaner's own half-written copy and is removed without +following a link: a link or a file at that name is unlinked as itself (what it leads to stays), and a folder is +removed by descriptors. Only then is the session folder deleted; a +copy that cannot be made or does not read back the same fails that delete (`FAILED ...: keep first: ...: nothing +deleted`, exit 1) and the folder stays whole. The run logs `kept first : files () in +(verified)` before its `deleting` line. The cleaner never deletes from the archive. + +`plan` shows both: each session it would delete, the files it would keep first under it (` would keep first + ()`), and a total (`agent-scratch: would keep first 2432 files (27.8 MB) in `). ## `compress-cold`: smaller, never gone @@ -326,8 +448,9 @@ frees nothing there; that is safe (nothing changes, the run exits 0). The copy g ACL it would get from its folder is removed. The config refuses a folder that is `/`, the home folder or above it, a must-keep path itself or a folder above one (`~/.codex` holds logins: name `~/.codex/sessions`), anything in or above `~/.claude*` or the folder a `~/.claude*` link leads to (Claude Code reads its transcripts as plain text), a -folder named like `*log-archive*` or inside one, the cleaner's own log or daily folder, a -folder another rule deletes from, or a folder in or above another of its folders. +folder named like `*log-archive*` or inside one, the cleaner's own log or daily folder (and, while `agent-scratch` +is on, its keep-first archive), a folder another rule deletes from (the `agent-scratch` root among them while that +rule is on), or a folder in or above another of its folders. **The plan** says `would compress (, compress-cold: not changed or read for N days ...)` for each file, and per folder group how much it would free: one zstd run over the start of every file (64 KB each, 16 MB in all) @@ -382,6 +505,11 @@ The `cleaner` section of the machine file ([common](../common/README.md)). A who "root": "/tmp", "patterns": { "example-build-*": 7, "example-crash-*": 1 } }, + "agentScratch": { + "enabled": true, + "days": 3, + "keepFirst": { "maxSessionBytes": 8388608 } + }, "daily": { "mode": "compare", "oldTimers": ["example-retention.timer"], @@ -397,26 +525,35 @@ The `cleaner` section of the machine file ([common](../common/README.md)). A who } ``` -| Key | Default | Meaning | -| ------------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------- | -| `logFile` | `~/.local/state/fork-cleaner/cleaner.log` | the log; its locks, journal and report mark are beside it (new, or the cleaner's own) | -| `logLines` | 5000 | lines the log keeps (10 or more) | -| `mustKeep` | none | more paths that are never deleted (`*` within one part) | -| `worktrees.repos[].root` | required | the folder the clones and worktrees are directly in | -| `worktrees.repos[].github` | required | `owner/name` on GitHub | -| `worktrees.repos[].regenerable` | none | plain names of ignored build folders, on top of `node_modules`, `__pycache__` and `.pytest_cache` | -| `worktrees.repos[].branches` | all | branch patterns (shell style, case as written; `*` crosses `/`): only these branches are taken | -| `worktrees.mergedHours` | 24 | hours after the merge (24 or more) | -| `worktrees.usedHours` | 24 | hours since the worktree or clone was last used (24 or more) | -| `docker.enabled` | false | turns the rule on | -| `docker.days` | 7 | age of an unused anonymous volume (7 or more) | -| `tmp.root` | `/tmp` | the temp folder | -| `tmp.patterns` | none | name pattern to days without a change (1 or more) | -| `daily.mode` | `compare` | what the timer does: `compare` (deletes nothing) or `run` | -| `daily.oldTimers` | none | timers (user or system) of the tools this one replaces: `run` refuses while one is on or unknown | -| `daily.outDir` | `~/.local/state/fork-cleaner/daily` | where `daily` writes its files | -| `daily.keepDays` | 30 | days `daily` keeps its dated files (1 or more) | -| `daily.compare[]` | none | the old tools' dry runs: `name`, `command` (absolute program, no shell), `pattern` (one group) | +| Key | Default | Meaning | +| ---------------------------------------- | ----------------------------------------- | ---------------------------------------------------------------------------------------------------- | +| `logFile` | `~/.local/state/fork-cleaner/cleaner.log` | the log; its locks, journal and report mark are beside it (new, or the cleaner's own) | +| `logLines` | 5000 | lines the log keeps (10 or more) | +| `mustKeep` | none | more paths that are never deleted (`*` within one part) | +| `worktrees.repos[].root` | required | the folder the clones and worktrees are directly in | +| `worktrees.repos[].github` | required | `owner/name` on GitHub | +| `worktrees.repos[].regenerable` | none | plain names of ignored build folders, on top of `node_modules`, `__pycache__` and `.pytest_cache` | +| `worktrees.repos[].branches` | all | branch patterns (shell style, case as written; `*` crosses `/`): only these branches are taken | +| `worktrees.mergedHours` | 24 | hours after the merge (24 or more) | +| `worktrees.usedHours` | 24 | hours since the worktree or clone was last used (24 or more) | +| `docker.enabled` | false | turns the rule on | +| `docker.days` | 7 | age of an unused anonymous volume (7 or more) | +| `tmp.root` | `/tmp` | the temp folder | +| `tmp.patterns` | none | name pattern to days without a change (1 or more) | +| `agentScratch.enabled` | false | turns the rule on | +| `agentScratch.root` | `/tmp/claude-{uid}` | the folder of the project folders; `{uid}` is the user's id; one user's own folder (see above) | +| `agentScratch.transcripts` | `["~/.claude*/projects/*"]` | the folders of the sessions' transcripts (patterns; at least one; one matching nothing keeps all) | +| `agentScratch.days` | 3 | days without a change, of the folder and of its transcript (3 or more) | +| `agentScratch.keepFirst.archive` | `~/.local/state/fork-cleaner/kept` | where the files kept first go (never in a folder the cleaner deletes from) | +| `agentScratch.keepFirst.patterns` | the list above | the reports kept first, relative to the session folder; at least one; a larger one keeps the session | +| `agentScratch.keepFirst.outputs` | `["tasks/*.output"]` | the command outputs kept first; at least one; a larger one is named in the manifest, not copied | +| `agentScratch.keepFirst.maxFileBytes` | 262144 | the largest file kept first (65536 or more) | +| `agentScratch.keepFirst.maxSessionBytes` | 8388608 | more than this to keep first: the session stays whole | +| `daily.mode` | `compare` | what the timer does: `compare` (deletes nothing) or `run` | +| `daily.oldTimers` | none | timers (user or system) of the tools this one replaces: `run` refuses while one is on or unknown | +| `daily.outDir` | `~/.local/state/fork-cleaner/daily` | where `daily` writes its files | +| `daily.keepDays` | 30 | days `daily` keeps its dated files (1 or more) | +| `daily.compare[]` | none | the old tools' dry runs: `name`, `command` (absolute program, no shell), `pattern` (one group) | An empty section (`"cleaner": {}`) is valid and does nothing. An unknown key, a wrong type, a number below its floor, a root folder that is `/`, the home folder or a folder above it: the command names the file and the key, @@ -520,7 +657,23 @@ The installer writes no machine file: the `cleaner` section is the owner's, writ total line. - **it stops half-way:** the entry it was deleting (temp folder, clone or worktree) is named `half deleted` by every later plan and run, never judged or deleted again, and the run exits 1 until a person acts; see - [Half-deleted entries](#half-deleted-entries) for what to do. + [Half-deleted entries](#half-deleted-entries) for what to do. An agent session's folder is judged again and + finished when every check passes and nothing in it changed since the delete began. +- **agent-scratch, keep first stopped half-way:** a half-written copy (`..partial`) is thrown away and + made again by the next run; a copy that was made but whose delete never ran is found, read back and used again + (no second copy); an older copy that differs stays, and the new one is `.2`, and so on. +- **agent-scratch, the session comes back between plan and delete:** the last check reads the process list, + the transcripts and the folder anew; it is `REFUSED` and nothing is copied or deleted. If it comes back while + keep first copies, the check after the copy refuses it: the copy stays in the archive (the next run uses it + again) and nothing is deleted. +- **agent-scratch, the session comes back during the delete:** a file it makes after the last check (its inode + time is later) stops the delete there, even when it got the inode number of a file just deleted; the rest + stays, `half deleted`, exit 1, and the next run sees "changed since the delete began". +- **agent-scratch, a half delete hit a hard-linked file:** unlinking one name of a file with another name changes + its inode time, which the next run reads as "changed since the delete began": the session stays `half deleted` + and the run exits 1 until a person removes it. This fails safe and is left so. +- **agent-scratch, the root is empty or missing:** nothing to do, exit 0. A machine file without + `agentScratch` leaves the rule off and the root is not even read. - **something is in use:** it is kept and named. A process, unit or pull request that arrives between plan and delete is seen by the last check before the delete. - **the config is old, empty or wrong:** an empty section does nothing; anything it does not understand stops the @@ -557,6 +710,7 @@ python3 -I -B fork-kit/cleaner/tests/worktrees.test.py python3 -I -B fork-kit/cleaner/tests/engine.test.py python3 -I -B fork-kit/cleaner/tests/report.test.py python3 -I -B fork-kit/cleaner/tests/compress.test.py +python3 -I -B fork-kit/cleaner/tests/scratch.test.py python3 -I -B fork-kit/cleaner/tests/daily.test.py python3 -I -B fork-kit/cleaner/tests/install.test.py ``` diff --git a/fork-kit/cleaner/lib/cleaner/compare.py b/fork-kit/cleaner/lib/cleaner/compare.py index e43ce4a873602..dbd339449106c 100644 --- a/fork-kit/cleaner/lib/cleaner/compare.py +++ b/fork-kit/cleaner/lib/cleaner/compare.py @@ -24,8 +24,8 @@ TIMEOUT = 1800 # seconds one old dry run may take NEW_ONLY = 3 # the exit code when the compare is complete and something is new only -WOULD = re.compile(r'^would delete (.+?) \((?:[0-9.]+ (?:B|KB|MB|GB)|size not known), (\w+): (.*)\)$') -KEPT = re.compile(r'^kept (.+?) \((\w+): ([^:)]+)(?:: (.*))?\)$') +WOULD = re.compile(r'^would delete (.+?) \((?:[0-9.]+ (?:B|KB|MB|GB)|size not known), ([\w-]+): (.*)\)$') +KEPT = re.compile(r'^kept (.+?) \(([\w-]+): ([^:)]+)(?:: (.*))?\)$') def _within(path, folder): diff --git a/fork-kit/cleaner/lib/cleaner/config.py b/fork-kit/cleaner/lib/cleaner/config.py index 9d458290e4c64..3d912575ad381 100644 --- a/fork-kit/cleaner/lib/cleaner/config.py +++ b/fork-kit/cleaner/lib/cleaner/config.py @@ -5,7 +5,7 @@ no key takes a refusal away, and the floors below cannot be lowered. An empty section means every rule does nothing. KEY_KINDS says, for every key, which of these it is; a test fails for a key that is not in it. """ -import fnmatch, glob, os, pwd, re +import fnmatch, glob, os, pwd, re, stat, tempfile import machine_file @@ -38,6 +38,7 @@ ('all docker volumes', 'rule:docker'), ('temp', 'rule:tmp'), ('cold folders', 'rule:compress-cold'), + ('agent scratch', 'rule:agent-scratch'), ('agent tool homes', AGENT_HOMES), ('T3 data', T3_DATA), ('build caches', None), @@ -56,6 +57,25 @@ # Folder names compress-cold never takes or enters (#468: the log archive stays as it is). `repair-backups`, the # other must-keep name, may be compressed: #468 names it. COMPRESS_NEVER_NAMES = ('*log-archive*',) +MIN_SCRATCH_DAYS = 3 # an agent session's scratch folder goes no sooner than this after anything in it changed +MIN_KEEP_FILE_BYTES = 64 * 1024 # keep first: files up to at least this size are copied before a delete +# Keep first (agentScratch.keepFirst): what is copied to the archive before a session's scratch folder goes. The +# default is chosen from a survey of real scratch folders: the small text reports an agent writes directly in its +# scratchpad, or one folder down (notes, results, verdicts), and not the clones and downloads deeper in it. +KEEP_FIRST_PATTERNS = ('scratchpad/*.md', 'scratchpad/*.json', 'scratchpad/*.txt', 'scratchpad/*.tsv', + 'scratchpad/*.csv', 'scratchpad/*/*.md') +# The full output of an agent's background commands (`tasks/.output`; a transcript holds only what the agent +# read back). Kept first up to maxFileBytes each; a larger one is not copied and does not hold the session, but the +# manifest names it with its size and sha256. +KEEP_FIRST_OUTPUTS = ('tasks/*.output',) +# Where agent tools keep a session's transcript, as folders of `.jsonl` files and `/` folders. Only their +# times are read. A link in the path is followed (a config folder that is a link), and every folder is used by its +# real path. +TRANSCRIPT_GLOBS = ('~/.claude*/projects/*',) +# Never an agent-scratch root: the shared temp folders themselves (every user's and every program's entries are +# directly in them). The root must be a folder of one user's own, such as /tmp/claude-. The user's own runtime +# folder (/run/user/, $XDG_RUNTIME_DIR) is refused too: every program of the user keeps its entries there. +SHARED_TEMP = ('/tmp', '/var/tmp', '/dev/shm', '/run/user', '/run/user/{uid}') # Folders any tool rebuilds, so an ignored one does not keep a worktree. A repository adds its own (regenerable). REGENERABLE = ('node_modules', '__pycache__', '.pytest_cache') @@ -120,6 +140,26 @@ def _mode(value): return None if value in DAILY_MODES else f'must be one of {", ".join(DAILY_MODES)} (not {value!r})' +def _globs_rel(value): + """Relative path patterns: parts split by /, * and ? within one part; no empty, . or .. part. At least one: + an empty list would turn keep first off, and no key may take that away.""" + if not value: + return 'must name at least one pattern (keep first cannot be turned off)' + for v in value: + if not isinstance(v, str) or not v or v.startswith('/') or '\0' in v: + return f'every entry must be a relative path pattern such as scratchpad/*.md (not {v!r})' + if any(part in ('', '.', '..') for part in v.split('/')): + return f'every entry must name files without an empty, . or .. part (not {v!r})' + return None + + +def _globs_abs(value): + """Folder patterns, absolute or from ~/, * and ? within one part. At least one.""" + if not value: + return 'must name at least one folder pattern' + return next((problem for problem in map(_path, value) if problem), None) + + def _github(value): return None if GITHUB_NAME.match(value) else f'must be owner/name (not {value!r})' @@ -131,6 +171,7 @@ def _github(value): 'worktrees': (dict, {}, None), 'docker': (dict, {}, None), 'tmp': (dict, {}, None), + 'agentScratch': (dict, {}, None), 'daily': (dict, {}, None), 'compressCold': (dict, {}, None), } @@ -153,6 +194,23 @@ def _github(value): 'root': (str, '/tmp', _path), 'patterns': (dict, {}, _patterns), # name pattern -> days without a change; the first match counts } +# The agent-scratch rule (rules.py): the scratch folders agent sessions leave in the temp folder, `// +# `. Off unless enabled. `{uid}` in root is the user's id (Claude Code's own folder is /tmp/claude-). +SCRATCH_SPEC = { + 'enabled': (bool, False, None), + 'root': (str, '/tmp/claude-{uid}', _path), + 'days': (int, MIN_SCRATCH_DAYS, _at_least(MIN_SCRATCH_DAYS)), + 'transcripts': (list, TRANSCRIPT_GLOBS, _globs_abs), # where the sessions' transcripts are (their times only) + 'keepFirst': (dict, {}, None), +} +# What is copied out of a session's folder before it is deleted, and where to (rules.keep_first). +KEEP_FIRST_SPEC = { + 'archive': (str, '~/.local/state/fork-cleaner/kept', _path), + 'patterns': (list, KEEP_FIRST_PATTERNS, _globs_rel), # relative to the session folder; never empty + 'outputs': (list, KEEP_FIRST_OUTPUTS, _globs_rel), # command output: a larger one is only named + 'maxFileBytes': (int, 256 * 1024, _at_least(MIN_KEEP_FILE_BYTES)), # a larger file is not a small report + 'maxSessionBytes': (int, 8 * 1024 * 1024, _at_least(0)), # more than this to keep first: the session stays whole +} # `fork-cleaner daily`, what the timer runs (daily.py). Without the section it plans, reports and compares. DAILY_SPEC = { 'mode': (str, 'compare', _mode), @@ -197,6 +255,15 @@ def _github(value): 'compare.command': 'compare only', 'compare.pattern': 'compare only', 'compressCold': 'picks', 'compressCold.folders': 'picks', 'compressCold.keepFreeGB': 'floor', 'folders.path': 'picks', 'folders.days': 'floor', + # agentScratch: on, its folder and its days pick what it looks at. keepFirst only adds a copy before a delete: + # more patterns copy more; maxFileBytes has a floor; a session with more than maxSessionBytes to copy stays + # whole, so the cap picks which sessions the rule may take at all (every refusal still holds for them). The +# transcript folders pick where a session's transcript is looked for; a folder pattern that matches nothing keeps +# every session ("could not read"), so a wrong entry keeps more, but leaving a real folder out would not. + 'agentScratch': 'picks', 'agentScratch.enabled': 'picks', 'agentScratch.root': 'picks', + 'agentScratch.days': 'floor', 'agentScratch.keepFirst': 'picks', 'agentScratch.transcripts': 'picks', + 'keepFirst.archive': 'own file', 'keepFirst.patterns': 'keeps more', 'keepFirst.outputs': 'keeps more', + 'keepFirst.maxFileBytes': 'floor', 'keepFirst.maxSessionBytes': 'picks', } @@ -205,6 +272,23 @@ def _holds(folder, path): return folder == '/' or path == folder or path.startswith(folder + '/') +def _transcript_overlap(root, globs): + """The transcript folder (pattern, or a folder it matches now) that the agent-scratch root is, is inside or + holds, by its path and by its real path; None when there is none. Agent logs are never deleted (README).""" + real = os.path.realpath(root) + for pattern in globs: + want = pattern.strip('/').split('/') + for path in (root, real): + have = path.strip('/').split('/') + if all(fnmatch.fnmatchcase(a, b) for a, b in zip(have, want)): + return pattern + for folder in sorted(glob.glob(pattern)): + folder = os.path.realpath(folder) + if os.path.isdir(folder) and (_holds(real, folder) or _holds(folder, real)): + return folder + return None + + def must_keep_match(patterns, path): """The must-keep pattern `path` is equal to, inside of or a folder above; None when there is none.""" parts = tuple(path.split('/')[1:]) @@ -282,6 +366,30 @@ def __init__(self, section, where=KIT, home=None): # Without a pattern the rule does nothing and its folder is no root, so the folder is not judged either. self.tmp_root = self._root(tmp['root'], f'{where}.tmp.root') if self.tmp_patterns else self._expand(tmp['root']) + scratch = machine_file.fields(values['agentScratch'], SCRATCH_SPEC, f'{where}.agentScratch') + keep = machine_file.fields(scratch['keepFirst'], KEEP_FIRST_SPEC, f'{where}.agentScratch.keepFirst') + self.scratch_enabled, self.scratch_days = scratch['enabled'], scratch['days'] + root_text = scratch['root'].replace('{uid}', str(os.getuid())) + if '{' in root_text or '}' in root_text: + raise machine_file.MachineFileError(f'{where}.agentScratch.root: only {{uid}} may stand in braces') + # Off, the rule looks at nothing and its folder is no root, so the folder is not judged either. + self.scratch_root = (self._scratch_root(root_text, f'{where}.agentScratch.root') if self.scratch_enabled + else self._expand(root_text)) + self.transcript_globs = tuple(self._expand(g) for g in scratch['transcripts']) + if self.scratch_enabled: + held = _transcript_overlap(self.scratch_root, self.transcript_globs) + if held is not None: + raise machine_file.MachineFileError(f'{where}.agentScratch.root: {root_text} overlaps the transcript ' + f'folder {held}: agent logs are never deleted') + self.keep_patterns = tuple(tuple(p.split('/')) for p in keep['patterns']) + self.keep_outputs = tuple(tuple(p.split('/')) for p in keep['outputs']) + self.keep_max_file, self.keep_max_session = keep['maxFileBytes'], keep['maxSessionBytes'] + archive = self._expand(keep['archive']) + self.keep_archive = os.path.join(os.path.realpath(os.path.dirname(archive)), os.path.basename(archive)) + if _holds(self.keep_archive, self.home): + raise machine_file.MachineFileError(f'{where}.agentScratch.keepFirst.archive: {keep["archive"]} is the ' + 'home folder or holds it') + docker = machine_file.fields(values['docker'], DOCKER_SPEC, f'{where}.docker') self.docker_enabled, self.docker_days = docker['enabled'], docker['days'] @@ -310,16 +418,20 @@ def __init__(self, section, where=KIT, home=None): if _holds(self.daily_dir, self.home): raise machine_file.MachineFileError(f'{where}.daily.outDir: {daily["outDir"]} is the home folder or holds it') - self.cold_folders = self._cold(values['compressCold'], f'{where}.compressCold', values['logFile'], daily['outDir']) + # The archive is written only while the rule is on: then it may not be where the cleaner deletes or + # compresses either. + archive = (('agentScratch.keepFirst.archive', keep['archive'], self.keep_archive),) if self.scratch_enabled else () + self.cold_folders = self._cold(values['compressCold'], f'{where}.compressCold', values['logFile'], daily['outDir'], + archive) - for root in self.roots(): + for root in self.roots() | ({self.scratch_root} if self.scratch_enabled else set()): for key, text, own in (('logFile', values['logFile'], self.log_file), - ('daily.outDir', daily['outDir'], self.daily_dir)): + ('daily.outDir', daily['outDir'], self.daily_dir)) + archive: if _holds(root, own) or _holds(own, root): raise machine_file.MachineFileError(f'{where}.{key}: {text} is inside {root}, a folder ' 'the cleaner deletes from') - def _cold(self, section, where, log_text, out_text): + def _cold(self, section, where, log_text, out_text, archive=()): """The folders of compress-cold. A folder may be inside a must-keep path (the rule keeps every file's content), but never one of them whole or a folder above one (`~/.codex` holds logins and settings; name `~/.codex/sessions`), never in or above COMPRESS_NEVER, the cleaner's own files or a folder another rule @@ -347,10 +459,11 @@ def _cold(self, section, where, log_text, out_text): if held: raise machine_file.MachineFileError(f'{at}.path: {text} is or holds the must-keep path /{"/".join(held[0])}: ' 'name a folder inside it (its settings and logins stay as they are)') - for key, own_text, own in (('logFile', log_text, self.log_file), ('daily.outDir', out_text, self.daily_dir)): + # `archive`: the keep-first archive of agent-scratch while that rule is on, also one of the cleaner's own files. + for key, own_text, own in (('logFile', log_text, self.log_file), ('daily.outDir', out_text, self.daily_dir)) + archive: if _holds(path, own) or _holds(own, path): raise machine_file.MachineFileError(f'{at}.path: {text} holds or is inside {key} {own_text}') - for root in self.roots(): + for root in self.roots() | ({self.scratch_root} if self.scratch_enabled else set()): if _holds(path, root) or _holds(root, path): raise machine_file.MachineFileError(f'{at}.path: {text} holds or is inside {root}, a folder ' 'another rule deletes from') @@ -385,13 +498,42 @@ def _root(self, text, where): raise machine_file.MachineFileError(f'{where}: {text} is the home folder or holds it') return root + def _scratch_root(self, text, where): + """The agent-scratch root: one user's own folder of project folders. Never a shared temp folder itself + (/tmp: every program's entries are directly in it), never / or home or above it, and, when it is there, a + real folder (not a link) that this user owns.""" + root = self._root(text, where) + runtime = (os.environ.get('XDG_RUNTIME_DIR') or '/run/user/{uid}',) + shared = {os.path.normpath(t.replace('{uid}', str(os.getuid()))) + for t in SHARED_TEMP + runtime + (tempfile.gettempdir(),) if t.startswith('/')} + if root in shared or os.path.realpath(root) in shared: + raise machine_file.MachineFileError(f'{where}: {text} is a shared temp folder; name the folder of one ' + 'user in it (such as /tmp/claude-{uid})') + try: + st = os.lstat(root) + except FileNotFoundError: + return root + except OSError as ex: + raise machine_file.MachineFileError(f'{where}: {text}: {ex.strerror}') from None + if not stat.S_ISDIR(st.st_mode): + raise machine_file.MachineFileError(f'{where}: {text} is a link or not a folder') + if st.st_uid != os.getuid(): + raise machine_file.MachineFileError(f'{where}: {text} is owned by uid {st.st_uid}, not by this user') + return root + def roots(self): - """The folders whose direct entries may be deleted at all. Nothing else is ever a candidate.""" + """The folders whose direct entries may be deleted at all, besides the session folders of agent scratch + (session_parent). Nothing else is ever a candidate.""" roots = {repo.root for repo in self.repos} if self.tmp_patterns: roots.add(self.tmp_root) return frozenset(roots) + def session_parent(self, folder): + """True when `folder` is a project folder directly in the agent-scratch root (the rule is on): its direct + entries, the session folders, may go. The root itself is not one: a project folder never goes.""" + return self.scratch_enabled and folder != self.scratch_root and os.path.dirname(folder) == self.scratch_root + def matches(name, globs): return any(fnmatch.fnmatchcase(name, glob) for glob in globs) diff --git a/fork-kit/cleaner/lib/cleaner/engine.py b/fork-kit/cleaner/lib/cleaner/engine.py index 656dd70d10c79..59b3bf82a1e46 100644 --- a/fork-kit/cleaner/lib/cleaner/engine.py +++ b/fork-kit/cleaner/lib/cleaner/engine.py @@ -28,8 +28,9 @@ fork-cleaner daily what the timer runs: plan, report and compare (cleaner.daily.mode "compare", the default), or run and report ("run", refused while a timer in cleaner.daily.oldTimers is on or unknown) -What may go is decided by three rules (worktrees, docker, tmp) and the "cleaner" section of the machine file; a -fourth (compress-cold) replaces cold files in the folders it names by a checked zstd copy, and deletes no content. +What may go is decided by four rules (worktrees, docker, tmp, agent-scratch) and the "cleaner" section of the machine +file; a fifth (compress-cold) replaces cold files in the folders it names by a checked zstd copy, and deletes no +content. What must stay is decided by the code and cannot be turned off. Exit code: 0 done, 1 something could not be read or deleted (it was kept), 2 wrong command or no usable machine file (nothing was done), 3 (compare and daily only) every step worked and the compare found paths only fork-cleaner would delete.''' @@ -80,16 +81,7 @@ def log_problem(fd=None, path=None): return None if STAMP.match(first) else 'its first line is not a line of the cleaner\'s log' -def human(n): - if n is None: - return 'size not known' - if n < 1024: - return f'{n} B' - for unit in ('KB', 'MB', 'GB'): - n /= 1024 - if n < 1024: - break - return f'{n:.1f} {unit}' +human = rules.human def sized(things): @@ -217,17 +209,24 @@ def write_journal(cfg, entries): raise LogError(f'the journal {cfg.journal_file}: {ex.strerror}') from None -def half_deleted(cfg, entries): - """(kept, left): a Kept for each path a delete started on that is still there, and the journal lines still - true. A volume, or a path that is gone, has nothing left to name.""" - kept, left = [], [] +def half_deleted(cfg, entries, facts=None): + """(kept, left, finish): a Kept for each path a delete started on that is still there, the journal lines still + true, and the items to finish. A volume, or a path that is gone, has nothing left to name. An agent session's + scratch folder is judged again (rules.finish_scratch): when every check passes for what is left, the run + finishes the delete; otherwise it is kept as half deleted, with the reason.""" + kept, left, finish = [], [], [] for entry in entries: if entry['kind'] == 'volume' or not os.path.lexists(entry['path']): continue left.append(entry) + again = rules.finish_scratch(cfg, facts, entry) if facts is not None and entry['rule'] == rules.SCRATCH else None + if isinstance(again, rules.Item): + finish.append(again) + continue + why = f'{again.code}: {again.detail}; ' if again is not None else '' kept.append(rules.Kept(entry['path'], entry['rule'], 'half deleted', - f'since {entry.get("at", "?")}: ' + HALF.format(journal=cfg.journal_file))) - return kept, left + f'since {entry.get("at", "?")}: {why}' + HALF.format(journal=cfg.journal_file))) + return kept, left, finish # --- deleting ----------------------------------------------------------------------------------------------------- @@ -241,6 +240,11 @@ def delete_all(cfg, facts, items, really, out, log=None, journal=None): journaled = item.kind not in rules.UNJOURNALED if not really: out(f'{would} {told}') + look = item.look + for name, size in (look.keep_first if look is not None else ()): + out(f' would keep first {item.path}/{name} ({human(size)})') + for name, size in (look.keep_named if look is not None else ()): + out(f' would name only {item.path}/{name} ({human(size)}, too large to copy)') continue facts.forget() # the last check reads the machine anew, not what the plan saw refusal = rules.recheck(cfg, facts, item) @@ -248,10 +252,32 @@ def delete_all(cfg, facts, items, really, out, log=None, journal=None): item.done, item.refusal = 'refused', refusal out(f'REFUSED {item.label} ({item.rule}: {refusal.code}{": " + refusal.detail if refusal.detail else ""})') continue + if item.kind == 'session': + # Keep first: what is worth keeping is copied out and read back before anything is deleted. A copy + # that fails keeps the folder whole. + try: + kept = rules.keep_first(cfg, item) + except rules.KeepFirstError as ex: + failed += 1 + item.done = 'failed' + out(f'FAILED {item.label} ({item.rule}: keep first: {ex}): nothing deleted') + continue + if kept is not None: + out(f'kept first {item.path}: {kept}') + # The copy took a while: is the session still gone? Read anew right before the first unlink. + facts.forget() + item.look.checked = rules.clock() # a file changed after this stops the delete (_clear) + refusal = rules.session_alive(cfg, facts, item.path, item.look) + if refusal is not None: + item.done, item.refusal = 'refused', refusal + out(f'REFUSED {item.label} ({item.rule}: {refusal.code}{": " + refusal.detail if refusal.detail else ""})') + continue log.write(one_line(f'{doing} {told}')) # a failed write stops the run before the delete - entry = {'kind': item.kind, 'path': item.path, 'rule': item.rule, - 'at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} - if journaled: + # A delete an earlier run began (a session folder it finishes now) keeps its first line until it is done. + prior = [e for e in journal if e['kind'] == item.kind and e['path'] == item.path] if journaled else [] + entry = prior[0] if prior else {'kind': item.kind, 'path': item.path, 'rule': item.rule, + 'at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())} + if journaled and not prior: journal.append(entry) write_journal(cfg, journal) try: @@ -259,9 +285,10 @@ def delete_all(cfg, facts, items, really, out, log=None, journal=None): except (OSError, Unreadable) as ex: failed += 1 item.done = 'failed' + gone = item.kind != 'volume' and not os.path.lexists(item.path) if not journaled: pass # its step leaves nothing half gone (rules.UNJOURNALED) - elif isinstance(ex, Untouched) or (item.kind != 'volume' and not os.path.lexists(item.path)): + elif gone or (isinstance(ex, Untouched) and not prior): journal.remove(entry) # nothing was removed, or all of it was: nothing is half gone write_journal(cfg, journal) out(f'FAILED {item.label} ({item.rule}: {ex})' + (' (half deleted: see the next plan)' if entry in journal else '')) @@ -307,17 +334,18 @@ def out(line): entries, unread_journal = [], True else: unread_journal = False - half, journal = half_deleted(cfg, entries) + half, journal, finish = half_deleted(cfg, entries, facts) if really and journal != entries: write_journal(cfg, journal) plan = rules.plan(cfg, facts, measure=not really) + plan.items += finish if unread_journal: # a plan only: anything may be half deleted, so nothing would be deleted plan.kept += [rules.Kept(item.path, item.rule, 'could not read', 'the journal of deletes in progress cannot ' 'be read, and it may name this as half deleted', item.size) for item in plan.items if item.kind != 'volume'] plan.items = [item for item in plan.items if item.kind == 'volume'] - halves = {kept.path for kept in half} - plan.items = [item for item in plan.items if item.kind == 'volume' or item.path not in halves] + halves = {kept.path for kept in half} | {item.path for item in finish} + plan.items = [item for item in plan.items if item.kind == 'volume' or item.path not in halves or item in finish] plan.kept = [kept for kept in plan.kept if kept.rule == 'docker' or kept.path not in halves] + half for kept in sorted(plan.kept, key=lambda k: (k.rule, k.path)): shown = f'docker volume {kept.path}' if kept.rule == 'docker' and kept.path != 'docker' else kept.path @@ -333,6 +361,10 @@ def out(line): words = rules.WORDS.get(kind, ('would delete', 'deleting', 'deleted')) say(f' {rule}: {words[2] if really else words[0]} {count(len(group), rules.NOUNS.get(kind, kind))} ' f'({_compressed(group, really) if kind == "compress" else sized(group)})') + first = [size for item in group for _, size in (item.look.keep_first if item.look is not None else ())] + if first: + say(f' {rule}: {"kept" if really else "would keep"} first {count(len(first), "file")} ({human(sum(first))}) ' + f'in {cfg.keep_archive}') for (rule, kind), group in _groups([i for i in plan.items if i.done == 'failed'], lambda i: (i.rule, i.kind)): say(f' {rule}: failed {count(len(group), rules.NOUNS.get(kind, kind))} ({sized(group)})') for (rule, code), group in _groups(plan.kept, lambda k: (k.rule, k.code)): diff --git a/fork-kit/cleaner/lib/cleaner/facts.py b/fork-kit/cleaner/lib/cleaner/facts.py index 738739dac9eb7..b80424a9935fa 100644 --- a/fork-kit/cleaner/lib/cleaner/facts.py +++ b/fork-kit/cleaner/lib/cleaner/facts.py @@ -4,7 +4,7 @@ and whoever asked keeps the thing it asked about. Nothing in this file changes anything, except remove_worktree and remove_volume at the very end, which the engine calls after every check has passed again. """ -import calendar, ipaddress, json, os, re, socket, stat, subprocess, time +import calendar, errno, fnmatch, ipaddress, json, os, re, socket, stat, subprocess, time UNIT_DIRS = ('/etc/systemd/system', '/run/systemd/system', '/run/systemd/transient', '/etc/systemd/user', '~/.config/systemd/user', '/run/user/{uid}/systemd/user', '/run/user/{uid}/systemd/transient') @@ -24,16 +24,45 @@ class Unreadable(Exception): # --- processes --------------------------------------------------------------------------------------------------- +# A word of a command line or an environment: what a session id is made of. A path splits into its parts. +WORD = re.compile(r'[A-Za-z0-9_.-]+') # Before and after a path named inside longer text: what may stand next to a whole path, not inside a longer name. EDGE = r'\s"\'=:;,&|()<>`' +class Proc: + """One process of the cleaner's own user. `paths`: what it works in or has open (its working folder, its root, + every open descriptor). `words`: the words of its command line and its environment. `start`: when it started + (seconds since the epoch; None when not known). `hidden`: a part of it could not be read although it is the + user's own (a process that is not dumpable, such as gpg-agent: its open files and environment are root's to + read). `name`: its short name (/proc//comm). `parent`: the pid of its parent (from /proc//stat, which + a hidden process shows too); None when not known.""" + + def __init__(self, pid, name='', paths=(), words=(), start=None, hidden=False, parent=None): + self.pid, self.name, self.paths, self.words = pid, name, set(paths), set(words) + self.start, self.hidden, self.parent = start, hidden, parent + + class Processes: """`busy`: every path a process uses, and every folder above each. `locks`: the inodes with a held file lock. - `texts`: the command lines as text, so a path with a space in it (`sh -c 'cd "/tmp/my dir"'`) is seen too.""" + `texts`: the command lines as text, so a path with a space in it (`sh -c 'cd "/tmp/my dir"'`) is seen too. + `words`: every word of every command line and of the environment of each process the user may read (a session + id: `--session-id=`, `CLAUDE_CODE_SESSION_ID=`, or a path with it). `own`: the user's own processes, + one Proc each (the agent-scratch rule asks which session folder each one works in).""" - def __init__(self, busy=(), locks=(), texts=()): + def __init__(self, busy=(), locks=(), texts=(), words=(), own=()): self.busy, self.locks, self.texts = set(busy), set(locks), list(texts) + self.words = set(words) + for text in self.texts: + self.words.update(WORD.findall(text)) + self.own = list(own) + for one in self.own: + self.words.update(one.words) + + def names(self, word): + """True when a process names `word` (a session id) as a whole word on its command line or in its + environment.""" + return word in self.words def uses(self, path): """True when a process uses `path` or something in it, or a command line names it.""" @@ -54,21 +83,6 @@ def ancestors(path): path = parent -def links(folder): - """Where the links in a /proc folder point; none when the folder cannot be read (another user's process).""" - out = [] - try: - names = os.listdir(folder) - except OSError: - return out - for name in names: - try: - out.append(os.readlink(os.path.join(folder, name))) - except OSError: - pass - return out - - def _command_paths(args, cwd): """Every path a command line may name: whole arguments, the value after `=`, and paths inside longer text (`sh -c "cd /x; ./run"`). A relative argument is read from the working folder. Too many is fine: a path named @@ -86,6 +100,36 @@ def _command_paths(args, cwd): return out +def _boot_time(proc): + try: + with open(os.path.join(proc, 'stat')) as f: + for line in f: + if line.startswith('btime '): + return float(line.split()[1]) + except (OSError, ValueError, IndexError): + pass + return None + + +def _stat(base, boot): + """(start, parent) of a process from /proc//stat: when it started (seconds since the epoch, field 22) and + its parent's pid (field 4); each None when not known.""" + try: + with open(os.path.join(base, 'stat'), 'rb') as f: + fields = f.read().rsplit(b')', 1)[1].split() + except (OSError, IndexError): + return None, None + try: + parent = int(fields[1]) + except (ValueError, IndexError): + parent = None + try: + start = None if boot is None else boot + int(fields[19]) / os.sysconf('SC_CLK_TCK') + except (ValueError, IndexError): + start = None + return start, parent + + def read_processes(proc='/proc', me=None): """Processes for everything now running. The cleaner sees its own user's processes in full and of other users what the kernel shows it (usually the command line). Raises Unreadable when the list, the lock list or the @@ -99,27 +143,64 @@ def read_processes(proc='/proc', me=None): socket_lines = f.read().splitlines()[1:] except OSError as ex: raise Unreadable(f'{ex.filename or proc}: {ex.strerror}') from None - paths, seen, texts = set(), set(), [] + paths, seen, texts, words, own = set(), set(), [], set(), [] + uid, boot = os.getuid(), _boot_time(proc) for pid in pids: base = os.path.join(proc, pid) + try: + mine = os.lstat(base).st_uid == uid + except OSError: + continue # it ended + one = Proc(int(pid)) cwd = None for name in ('cwd', 'root', 'exe'): try: target = os.readlink(os.path.join(base, name)) - except OSError: + except OSError as ex: + one.hidden = one.hidden or (mine and name != 'exe' and ex.errno in (errno.EACCES, errno.EPERM)) continue - paths.add(target) + paths.add(target) # its folder, root or program + if name != 'exe': + one.paths.add(target) if name == 'cwd': cwd = target seen.add(int(pid)) - paths.update(links(os.path.join(base, 'fd'))) + try: + fds = os.listdir(os.path.join(base, 'fd')) + except OSError as ex: + fds = [] + one.hidden = one.hidden or (mine and ex.errno in (errno.EACCES, errno.EPERM)) + for name in fds: + try: + target = os.readlink(os.path.join(base, 'fd', name)) + except OSError: + continue + paths.add(target) # an open file + one.paths.add(target) try: with open(os.path.join(base, 'cmdline'), 'rb') as f: args = [a.decode('utf-8', 'surrogateescape') for a in f.read().split(b'\0') if a] paths.update(_command_paths(args, cwd)) texts.append(' '.join(args)) + one.words.update(WORD.findall(' '.join(args))) except OSError: pass + try: + # Only the user's own processes show it: what an agent's tools inherit (its session id) is here. + with open(os.path.join(base, 'environ'), 'rb') as f: + found = WORD.findall(f.read().decode('utf-8', 'surrogateescape')) + words.update(found) + one.words.update(found) + except OSError as ex: + one.hidden = one.hidden or (mine and ex.errno in (errno.EACCES, errno.EPERM)) + if mine: + try: + with open(os.path.join(base, 'comm'), errors='replace') as f: + one.name = f.read().strip() + except OSError: + pass + one.start, one.parent = _stat(base, boot) + own.append(one) try: with open(os.path.join(base, 'maps'), errors='surrogateescape') as f: for line in f: @@ -150,7 +231,7 @@ def read_processes(proc='/proc', me=None): found = LOCK_INODE.match(field) if found: locks.add(int(found.group(1))) - return Processes(busy, locks, texts) + return Processes(busy, locks, texts, words, own) # --- mounts ------------------------------------------------------------------------------------------------------- @@ -639,6 +720,63 @@ def docker_volume_sizes(): return sizes, folder +# --- agent transcripts ------------------------------------------------------------------------------------------- +# Where agent tools keep a session's transcript, named after its id: `.jsonl`, and a folder `` of its +# sub-agents. The folders are the machine file's (agentScratch.transcripts). Only their times are read (never their +# content), and nothing here changes them: they are must-keep. +def transcript_folders(globs): + """The real paths of the folders the patterns match (a link on the way, such as a config folder that is a link, + is followed). Raises Unreadable when a folder on the way cannot be read, or a pattern matches no folder at all + (a transcript folder that is not there could be one that moved: no answer is no "no transcript").""" + out = [] + for pattern in globs: + found = ['/'] + for part in pattern.strip('/').split('/'): + nxt = [] + for base in found: + if not any(c in part for c in '*?['): + path = os.path.join(base, part) + if os.path.isdir(path): + nxt.append(path) + continue + try: + names = sorted(os.listdir(base)) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as ex: + raise Unreadable(f'{base}: {ex.strerror}') from None + for name in names: + path = os.path.join(base, name) + if fnmatch.fnmatchcase(name, part) and os.path.isdir(path): + nxt.append(path) + found = nxt + if not found: + raise Unreadable(f'no folder matches {pattern}') + for folder in found: + real = os.path.realpath(folder) + if real not in out: + out.append(real) + return out + + +def read_transcripts(globs): + """{session id: the newest change of its transcript files}. Raises Unreadable when a folder cannot be read or a + pattern matches no folder (transcript_folders).""" + out = {} + for folder in transcript_folders(globs): + try: + with os.scandir(folder) as it: + for entry in it: + sid = entry.name[:-len('.jsonl')] if entry.name.endswith('.jsonl') else entry.name + st = entry.stat(follow_symlinks=False) + out[sid] = max(out.get(sid, 0.0), st.st_mtime, st.st_ctime) + except FileNotFoundError: + continue + except OSError as ex: + raise Unreadable(f'{ex.filename or folder}: {ex.strerror}') from None + return out + + # --- one view of it all ------------------------------------------------------------------------------------------- class Facts: """What one run knows. While planning, the process list and the mounts are read again when they are older than @@ -646,18 +784,19 @@ class Facts: it before the last check of each delete, so that check reads the process list, the mounts, the units, the pull request lists and Docker's list anew. A reader that failed keeps failing until then.""" - def __init__(self, cfg, processes=None, units=None, prs=None, dangling=None, now=None, mounts=None): + def __init__(self, cfg, processes=None, units=None, prs=None, dangling=None, now=None, mounts=None, transcripts=None): self.cfg = cfg self.now = time.time() if now is None else now self._read_processes = processes or read_processes self._read_mounts = mounts or read_mounts self._read_units = units or (lambda: read_units(home=cfg.home)) + self._read_transcripts = transcripts or (lambda: read_transcripts(cfg.transcript_globs)) self._read_prs = prs or github_prs self.dangling = dangling or docker_dangling self.forget() def forget(self): - self._processes = self._mounts = self._units = None + self._processes = self._mounts = self._units = self._transcripts = None self._prs = {} @staticmethod @@ -683,6 +822,11 @@ def mounts(self): self._mounts = (time.monotonic(), self._answer(self._read_mounts)) return self._give(self._mounts[1]) + def transcripts(self): + if self._transcripts is None: + self._transcripts = (self._answer(self._read_transcripts),) + return self._give(self._transcripts[0]) + def units(self): if self._units is None: self._units = (self._answer(self._read_units),) diff --git a/fork-kit/cleaner/lib/cleaner/guard.py b/fork-kit/cleaner/lib/cleaner/guard.py index cbdf98e0e77e0..e4f3de0bb1504 100644 --- a/fork-kit/cleaner/lib/cleaner/guard.py +++ b/fork-kit/cleaner/lib/cleaner/guard.py @@ -37,7 +37,12 @@ class Look: def __init__(self, refusal=None, size=0, newest=0.0): self.refusal, self.size, self.newest = refusal, size, newest + self.newest_file = 0.0 # the newest change of anything in it that is not a folder (a delete changes folders) + self.keep_first = [] # agent-scratch: (relative path, size) of what is copied to the archive first + self.keep_named = [] # agent-scratch: (relative path, size) of the outputs too large to copy: named only self.ident, self.seen = None, set() + self.linked = set() # the inodes in `seen` of files with several hard links + self.checked = None # agent-scratch: when the last liveness check began (rules.clock); see _clear def changed(st): @@ -183,7 +188,7 @@ def _in_use(facts, path, inodes): def _check(cfg, facts, path, kind, look): root = os.path.dirname(path) - if os.path.dirname(path) not in cfg.roots() or os.path.normpath(path) != path: + if not (root in cfg.roots() or cfg.session_parent(root)) or os.path.normpath(path) != path: return Refusal('not in a configured root', 'only what is directly in a configured folder may go') try: st, rst = os.lstat(path), os.lstat(root) @@ -265,11 +270,14 @@ def _check(cfg, facts, path, kind, look): inodes.append((est.st_ino, shown)) look.seen.add(est.st_ino) look.newest = max(look.newest, changed(est)) + if not stat.S_ISDIR(est.st_mode): + look.newest_file = max(look.newest_file, changed(est)) twice = est.st_nlink > 1 and not stat.S_ISDIR(est.st_mode) # a hard link counts once if not twice or (est.st_dev, est.st_ino) not in counted: look.size += est.st_blocks * 512 if twice: counted.add((est.st_dev, est.st_ino)) + look.linked.add(est.st_ino) if is_device(est): return Refusal('not a file or folder', f'{shown} is a device') if plain and entry.name == '.git': diff --git a/fork-kit/cleaner/lib/cleaner/report.py b/fork-kit/cleaner/lib/cleaner/report.py index 367f669ef7d5e..6d8daa7764963 100644 --- a/fork-kit/cleaner/lib/cleaner/report.py +++ b/fork-kit/cleaner/lib/cleaner/report.py @@ -30,7 +30,7 @@ DELETED = re.compile(r' \((\d+ B|\d+\.\d [KMG]B|size not known), (' + '|'.join(RULES) + r'): ') UNITS = {'B': 1, 'KB': 1024, 'MB': 1024 ** 2, 'GB': 1024 ** 3} # The other lines a run writes to the log (engine.py): understood, and not a delete. -OTHER_LINES = ('deleting ', 'compressing ', 'FAILED ', 'REFUSED ', 'total: ', 'stopped: ') +OTHER_LINES = ('deleting ', 'compressing ', 'kept first ', 'FAILED ', 'REFUSED ', 'total: ', 'stopped: ') MARK_VERSION = 1 @@ -182,6 +182,17 @@ def measure(cfg, sizer, name, source, docker): for folder in cfg.cold_folders: # without folders the rule looks at nothing if os.path.lexists(folder.path): sized.add(sizer.size(folder.path)) + elif source == 'rule:agent-scratch': + if not cfg.scratch_enabled: + sized.note = 'off' # as in a plan: the rule looks at nothing + else: + for project in _names(cfg.scratch_root, sized): + folder = os.path.join(cfg.scratch_root, project) + if os.path.islink(folder) or not os.path.isdir(folder): + continue + for entry in _names(folder, sized): + if rules.SESSION_NAME.match(entry): + sized.add(sizer.size(os.path.join(folder, entry))) elif source == 'rule:docker': if not cfg.docker_enabled: sized.note = 'off' # as in a plan: Docker is not even asked diff --git a/fork-kit/cleaner/lib/cleaner/rules.py b/fork-kit/cleaner/lib/cleaner/rules.py index a95479df927bf..8016b04b53eef 100644 --- a/fork-kit/cleaner/lib/cleaner/rules.py +++ b/fork-kit/cleaner/lib/cleaner/rules.py @@ -1,21 +1,21 @@ -"""The three rules: `worktrees`, `docker` and `tmp`. A rule only names what it wants gone and why; whether a path +"""The four rules: `worktrees`, `docker`, `tmp` and `agent-scratch`. A rule only names what it wants gone and why; whether a path may go at all is the guard's answer (guard.py), and every rule asks it. `plan` reads and decides, and changes nothing. `recheck` reads everything about one item again, just before its delete. `delete` is the only code in the kit that removes anything. """ -import errno, fnmatch, os, re, stat +import calendar, errno, fnmatch, hashlib, json, os, re, shutil, stat, sys, time from . import compress, guard from .config import matches -from .facts import (Unreadable, Untouched, default_branch, elsewhere, git, git_code, local_tags, lonely, merged_seconds, +from .facts import (DELETED, Unreadable, Untouched, default_branch, elsewhere, git, git_code, local_tags, lonely, merged_seconds, origin_url, orphaned_worktree, present, reflog, remote_refs, remote_urls, remove_volume, remove_worktree, worktree_list) from .guard import CREDENTIAL_NAMES, Refusal DAY = 86400 # Names in the temp folder no pattern can take: hidden entries (sockets of the desktop), what the system or a -# login session owns, and agent scratch folders (their own rule, which needs the owner's yes, is not built yet). +# login session owns, and agent scratch folders (the agent-scratch rule's, never the tmp rule's). TMP_NEVER = ('.*', 'claude-*', 'tmux-*', 'ssh-*', 'systemd-private-*', 'snap-private-tmp', 'snap.*', 'lost+found') ANONYMOUS = re.compile(r'^[0-9a-f]{64}$') # the name Docker gives a volume nobody named EMPTY_PRS = {'merged': [], 'open': []} @@ -52,6 +52,22 @@ def __init__(self): self.items, self.kept = [], [] +def human(n): + if n is None: + return 'size not known' + if n < 1024: + return f'{n} B' + for unit in ('KB', 'MB', 'GB'): + n /= 1024 + if n < 1024: + break + return f'{n:.1f} {unit}' + + +def files_text(n): + return f'{n} file' if n == 1 else f'{n} files' + + def days_text(n): return f'{n} day' if n == 1 else f'{n} days' @@ -118,6 +134,541 @@ def plan_tmp(cfg, facts, plan, measure): plan.items.append(Item('path', path, 'tmp', why, look.size, look)) +# --- agent-scratch ------------------------------------------------------------------------------------------------ +# The scratch folder of one agent session is `//` (Claude Code: /tmp/claude-//). Only folders named like a session id are looked at. +SESSION_NAME = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$') +SCRATCH = 'agent-scratch' +MANIFEST = 'manifest.json' +NO_FOLLOW = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC + + +class KeepFirstError(Exception): + """The copy into the archive could not be made or verified: the session folder stays.""" + + +def _stamp(seconds): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime(seconds)) + + +def _seconds(stamp): + try: + return float(calendar.timegm(time.strptime(stamp, '%Y-%m-%dT%H:%M:%SZ'))) + except (TypeError, ValueError, OverflowError): + return None + + +class Found: + """What keep first finds in a session folder. `keep`: (name, size, source) of each file it copies (`source`: + the parts of the path it is read from, inside the session folder; for an output that is a link, where the link + leads). `held`: (name, size) of each report over maxFileBytes, which keeps the session whole (it is not copied, + and nothing may be deleted before it is). `listed`: (name, size, source) of each command output over + maxFileBytes: not copied, and named in the manifest with its sha256. `out`: (name, link text) of each output + that is a link out of the session folder (to a sub-agent's transcript, which stays): never followed.""" + + def __init__(self): + self.keep, self.held, self.listed, self.out = [], [], [], [] + + +def _matching(path, pattern, look): + """(entry path, relative parts, lstat) of each entry the pattern names in the session folder: folders on the + way are real folders the last check walked, never a link, never a git folder or a clone.""" + bases = [(path, ())] + for depth, glob in enumerate(pattern): + last = depth == len(pattern) - 1 + nxt = [] + for base, rel in bases: + try: + with os.scandir(base) as it: + entries = list(it) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as ex: + raise Unreadable(f'{base}: {ex.strerror}') from None + for entry in entries: + if entry.name == '.git' or not fnmatch.fnmatchcase(entry.name, glob): + continue + st = entry.stat(follow_symlinks=False) + if st.st_ino not in look.seen: + continue + if last: + nxt.append((entry.path, rel + (entry.name,), st)) + elif stat.S_ISDIR(st.st_mode) and not os.path.lexists(os.path.join(entry.path, '.git')): + nxt.append((entry.path, rel + (entry.name,))) + bases = nxt + return sorted(bases, key=lambda b: b[1]) + + +def _inside(path, entry_path, look): + """For an output that is a link: (parts, lstat) of the regular file it leads to when that is in the session + folder and the last check walked it; None when it leads out (or to anything else).""" + if os.path.realpath(path) != path: + return None + target = os.path.realpath(entry_path) + if not target.startswith(path + '/'): + return None + try: + st = os.lstat(target) + except OSError: + return None + if not stat.S_ISREG(st.st_mode) or st.st_ino not in look.seen: + return None + return tuple(target[len(path) + 1:].split('/')), st + + +def keep_first_files(cfg, path, look): + """What keep first copies, holds or names in a session folder (Found). The machine file's report patterns: + regular files only, never through a link, never inside a git folder; a report over maxFileBytes holds the + session. Its output patterns: the same, and a link is resolved only to a regular file inside the session + folder; an output over maxFileBytes is named, not copied.""" + found, names = Found(), set() + for outputs, patterns in ((False, cfg.keep_patterns), (True, cfg.keep_outputs)): + for pattern in patterns: + for entry_path, rel, st in _matching(path, pattern, look): + name = '/'.join(rel) + if name in names: + continue + source = rel + if outputs and stat.S_ISLNK(st.st_mode): + resolved = _inside(path, entry_path, look) + if resolved is None: + try: + found.out.append((name, os.readlink(entry_path))) + except OSError: + found.out.append((name, '?')) + names.add(name) + continue + source, st = resolved + if not stat.S_ISREG(st.st_mode): + continue + names.add(name) + if st.st_size <= cfg.keep_max_file: + found.keep.append((name, st.st_size, source)) + elif outputs: + found.listed.append((name, st.st_size, source)) + else: + found.held.append((name, st.st_size)) + found.keep.sort() + return found + + +def _open_session(item): + """A descriptor of the session folder, opened one name at a time from /, never through a link, and only when + it is still the folder the last check judged.""" + try: + parent = _open_parent(item.path) + try: + top = os.open(os.path.basename(item.path), DIR_FLAGS, dir_fd=parent) + finally: + os.close(parent) + except OSError as ex: + raise KeepFirstError(f'{item.path}: {ex.strerror}') from None + st = os.fstat(top) + if item.look is None or item.look.ident != (st.st_dev, st.st_ino): + os.close(top) + raise KeepFirstError(f'{item.path} is not what the last check saw') + return top + + +def _open_in(top, parts, look): + """A descriptor of the file `parts` in the open session folder: every folder on the way entered by descriptor + and never through a link, the file opened without following a link, and only a regular file the last check + walked (its inode).""" + fd = top + try: + for part in parts[:-1]: + new = os.open(part, DIR_FLAGS, dir_fd=fd) + if fd != top: + os.close(fd) + fd = new + out = os.open(parts[-1], NO_FOLLOW, dir_fd=fd) + except OSError as ex: + raise KeepFirstError(f'{"/".join(parts)}: {ex.strerror}') from None + finally: + if fd != top: + os.close(fd) + st = os.fstat(out) + if not stat.S_ISREG(st.st_mode) or st.st_ino not in look.seen: + os.close(out) + raise KeepFirstError(f'{"/".join(parts)}: not the regular file the last check saw') + return out + + +def _read_fd(fd, sha=None): + """The whole file (or, with `sha`, only its size, the content fed to `sha`); closes the descriptor.""" + try: + chunks, size = [], 0 + while True: + chunk = os.read(fd, 1 << 20) + if not chunk: + return size if sha is not None else b''.join(chunks) + size += len(chunk) + if sha is None: + chunks.append(chunk) + else: + sha.update(chunk) + finally: + os.close(fd) + + +def _read_whole(path): + return _read_fd(os.open(path, NO_FOLLOW)) + + +def _sha(data): + return hashlib.sha256(data).hexdigest() + + +def _read_manifest(folder): + """The manifest of an archived copy as ({path: sha256} of the copied files, {path: sha256} of the files only + named); None when there is none or it cannot be used.""" + try: + manifest = json.loads(_read_whole(os.path.join(folder, MANIFEST))) + return ({f['path']: f['sha256'] for f in manifest['files']}, + {f['path']: f['sha256'] for f in manifest.get('notCopied', [])}) + except (OSError, ValueError, KeyError, TypeError, AttributeError): + return None + + +def _verified(folder, wanted, named=None): + """True when the archived copy in `folder` (a real folder, never a link) holds every (path, sha256) of + `wanted`, each file read back, and names every (path, sha256) of `named`.""" + try: + if not stat.S_ISDIR(os.lstat(folder).st_mode): + return False + except OSError: + return False + have = _read_manifest(folder) + if have is None or any(have[0].get(name) != sha for name, sha in wanted.items()): + return False + if any(have[1].get(name) != sha for name, sha in (named or {}).items()): + return False + try: + return all(_sha(_read_whole(os.path.join(folder, *name.split('/')))) == sha for name, sha in wanted.items()) + except OSError: + return False + + +def _sync_folder(folder): + fd = os.open(folder, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _own_folder(path, top): + """Makes the folder `path` and every missing folder above it (mode 0700), each synced into its parent, so a + copy in it outlives a power cut. Every folder from `top` (the archive) down to `path` must then be a real folder + of this user: the copy is never written through a link.""" + missing, here = [], path + while not os.path.lexists(here): + missing.append(here) + here = os.path.dirname(here) + for folder in reversed(missing): + try: + os.mkdir(folder, 0o700) + except FileExistsError: + pass + _sync_folder(os.path.dirname(folder)) + here = path + while True: + st = os.lstat(here) + if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid(): + raise KeepFirstError(f'{here}: a link, not a folder, or not this user\'s: no copy is written there') + if here == top: + return + here = os.path.dirname(here) + + +def keep_first(cfg, item): + """Copies the files worth keeping (keep_first_files) of a session folder into the archive, verifies the copy + and returns what it says about it; None when there is nothing to keep. Raises KeepFirstError when the copy + cannot be made or read back, or a report is too large to copy: the folder then stays. Runs again safely: a copy + already in the archive that holds the same files with the same sha256 is used as it is; a half-written copy + (`..partial`, the cleaner's own) is thrown away and made again; a different older copy stays, and the + new one gets the next free name.""" + try: + found = keep_first_files(cfg, item.path, item.look) + except Unreadable as ex: + raise KeepFirstError(str(ex)) from None + if found.held: + raise KeepFirstError(f'{found.held[0][0]}: larger than maxFileBytes: the session stays') + if ([(name, size) for name, size, _ in found.keep] != list(item.look.keep_first) or + [(name, size) for name, size, _ in found.listed] != list(item.look.keep_named)): + raise KeepFirstError('what is worth keeping is not what the last check saw: the session stays') + if not found.keep and not found.listed: + return None + project, session = item.path.split('/')[-2:] + data, named, sizes = {}, {}, {} + top = _open_session(item) + try: + for name, _, source in found.keep: + data[name] = _read_fd(_open_in(top, source, item.look)) + for name, _, source in found.listed: + sha = hashlib.sha256() + sizes[name] = _read_fd(_open_in(top, source, item.look), sha) + named[name] = sha.hexdigest() + except OSError as ex: + raise KeepFirstError(f'{ex.filename or item.path}: {ex.strerror}') from None + finally: + os.close(top) + wanted = {name: _sha(body) for name, body in data.items()} + total = sum(len(body) for body in data.values()) + folder = os.path.join(cfg.keep_archive, project) + final = os.path.join(folder, session) + told = f'{files_text(len(data))} ({human(total)})' + (f', {len(named)} larger named only' if named else '') + try: + _own_folder(folder, cfg.keep_archive) + versions = [final] + [f'{final}.{n}' for n in range(2, 100)] + for version in versions: + if not os.path.lexists(version): + break + if _verified(version, wanted, named): + return f'{told} already in {version} (verified)' + else: + raise KeepFirstError(f'{final}: 99 different copies are there already') + partial = os.path.join(folder, f'.{session}.partial') + if os.path.lexists(partial): + _remove_own(partial) + os.mkdir(partial, 0o700) + for name, body in data.items(): + target = os.path.join(partial, *name.split('/')) + os.makedirs(os.path.dirname(target), mode=0o700, exist_ok=True) + fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.write(fd, body) + os.fsync(fd) + finally: + os.close(fd) + manifest = {'version': 1, 'rule': SCRATCH, 'source': item.path, 'project': project, 'session': session, + 'keptAt': _stamp(time.time()), + 'files': [{'path': name, 'bytes': len(data[name]), 'sha256': wanted[name]} for name in sorted(data)], + 'notCopied': [{'path': name, 'bytes': sizes[name], 'sha256': named[name], + 'why': f'larger than maxFileBytes ({cfg.keep_max_file})'} for name in sorted(named)], + 'linksOut': [{'path': name, 'to': to, 'why': 'a link out of the session folder: not followed'} + for name, to in found.out]} + fd = os.open(os.path.join(partial, MANIFEST), os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.write(fd, (json.dumps(manifest, indent=1) + '\n').encode()) + os.fsync(fd) + finally: + os.close(fd) + for here, _, _ in os.walk(partial): + _sync_folder(here) + if not _verified(partial, wanted, named): + raise KeepFirstError(f'{partial}: the copy does not read back the same') + os.rename(partial, version) + _sync_folder(folder) + except OSError as ex: + raise KeepFirstError(f'{ex.filename or folder}: {ex.strerror}') from None + if not _verified(version, wanted, named): + raise KeepFirstError(f'{version}: the copy does not read back the same') + return f'{told} in {version} (verified)' + + +def _remove_own(partial): + """Removes a half-written copy of the cleaner's own from the archive, never through a link: a link or a file at + its name is unlinked as itself (whatever it leads to stays), and a folder is removed by descriptors, which + refuse a link at the top and never follow one below (shutil.rmtree's fd-based walk).""" + if not shutil.rmtree.avoids_symlink_attacks: + raise KeepFirstError(f'{partial}: this system cannot remove a folder without following links') + st = os.lstat(partial) + if not stat.S_ISDIR(st.st_mode): + os.unlink(partial) + return + shutil.rmtree(partial) + + +# Processes of the user that hide their open files and environment on purpose (not dumpable, so only root may +# read them) and hold no agent session: a login session's helpers and the key agents. The name excuses only what +# cannot be read: what can be read of such a process (its command line, its sockets) still keeps a session. Any +# other hidden process of the user is judged by its nearest readable ancestor (_judged_by); one with none keeps +# every session ("could not read"). Compared with /proc//comm. +HIDDEN_OK = ('systemd', '(sd-pam)', 'gpg-agent', 'ssh-agent', 'scdaemon', 'keyboxd', 'gnome-keyring-d') +# The user's service manager (systemd --user): where an ancestor walk stops, as at pid 1. A hidden process started +# by it, or by pid 1, has no agent above it to be judged by. +USER_MANAGER = 'systemd' + + +def _judged_by(p, own): + """The nearest ancestor of the hidden process `p` that can be read in full and is the user's own: what it works + in, has open and names decides for `p` too (a Codex started by an agent's tool hides itself, and its parent + names the agent's session). None when there is none short of pid 1 or the user's service manager, or an + ancestor is not the user's, has ended or cannot be told.""" + by_pid = {q.pid: q for q in own} + walked, pid = set(), p.parent + while pid is not None and pid > 1 and pid not in walked: + walked.add(pid) + q = by_pid.get(pid) + if q is None or q.name == USER_MANAGER: + return None + if not q.hidden: + return q + pid = q.parent + return None + + +def session_alive(cfg, facts, path, look): + """A refusal when the session may still be alive (README, agent-scratch): its transcript changed within + `days`; a process works in its folder, has something in it open or names its path; a process names its id; a + process of the user cannot be read in full; or a process works in another session of the same project without + naming that one, and started before this session last changed (after /clear or /resume a session's process + works on under a new id). None when it is gone. Asked as late as possible: the plan and the last check ask it + last, and the run asks it again after keep first, just before the delete.""" + sid, project = os.path.basename(path), os.path.dirname(path) + days = cfg.scratch_days + try: + transcripts = facts.transcripts() + except Unreadable as ex: + return Refusal('could not read', f'the transcripts ({ex})') + said = transcripts.get(sid) + if said is not None and facts.now - said < days * DAY: + return Refusal(f'transcript changed under {days_text(days)} ago', f'{_stamp(said)}') + try: + processes = facts.processes() + except Unreadable as ex: + return Refusal('could not read', f'the process list ({ex})') + if processes.uses(path): + return Refusal('session alive', 'a process works in it, has a file or folder in it open, or names its path') + if processes.names(sid): + return Refusal('session alive', 'a process names its id (command line or environment)') + # A hidden process with a readable ancestor is that ancestor's: the ancestor's command line, environment, working + # folder and open files were asked above and are asked below, like every other process. + unread = [p for p in processes.own if p.hidden and p.name not in HIDDEN_OK and _judged_by(p, processes.own) is None] + if unread: + return Refusal('could not read', f'process {unread[0].pid} ({unread[0].name or "?"}) of this user does not ' + 'show its open files or environment') + newest = max(look.newest if look is not None else 0.0, said or 0.0) + for p in processes.own: + for where in sorted(p.paths): + if where.endswith(DELETED): + where = where[:-len(DELETED)] + if not where.startswith(project + '/'): + continue + other = where[len(project) + 1:].split('/')[0] + if other == sid or (SESSION_NAME.match(other) and other in p.words): + continue # this one (uses), or a session the process names: it is that one's + if p.start is None or p.start <= newest: + return Refusal('session alive', f'process {p.pid} ({p.name or "?"}) works in {other} of the same ' + 'project without naming it, and started before this session last changed') + return None + + +def scratch_answer(cfg, facts, path, started=None): + """(why, look) for one session folder, or None when the rule does not take it (changed too recently). + `started`: an earlier run began to delete it at this time (seconds); then the folders' own times do not count + (that delete changed them), and nothing that is not a folder may have changed since.""" + sid = os.path.basename(path) + if not SESSION_NAME.match(sid): + return None + try: + top = guard.changed(os.lstat(path)) + except FileNotFoundError: + return None + except OSError as ex: + return '', guard.Look(Refusal('could not read', f'{path}: {ex.strerror}')) + days = cfg.scratch_days + look = guard.examine(cfg, facts, path, 'path') + if started is None: + age = facts.now - max(top, look.newest) + if age < days * DAY: + return None + why = f'session gone, not changed for {days_text(int((age + 1) // DAY))}' + else: + if look.newest_file > started: + look.refusal = look.refusal or Refusal('changed since the delete began', f'after {_stamp(started)}') + why = f'finishing the delete an earlier run began at {_stamp(started)}, session gone' + if look.refusal is not None: + return why, look + # A lock-named file directly in the session folder. Deeper, `yarn.lock` or `Cargo.lock` of a clone is ordinary + # content; a lock a process holds anywhere in it is the guard's `in use`. + try: + with os.scandir(path) as it: + locks = sorted(entry.name for entry in it if guard.matches(entry.name, guard.LOCK_NAMES)) + except OSError as ex: + look.refusal = Refusal('could not read', f'{path}: {ex.strerror}') + return why, look + if locks: + look.refusal = Refusal('lock file', some(locks)) + return why, look + try: + found = keep_first_files(cfg, path, look) + except Unreadable as ex: + look.refusal = Refusal('could not read', str(ex)) + return why, look + if found.held: + look.refusal = Refusal('report too large to keep first', + f'{some([name for name, _ in found.held])}: more than maxFileBytes ' + f'({human(cfg.keep_max_file)}); it stays until a person keeps it or it is gone') + return why, look + total = sum(size for _, size, _ in found.keep) + if total > cfg.keep_max_session: + look.refusal = Refusal('too much to keep first', f'{files_text(len(found.keep))}, {human(total)}: more than ' + f'maxSessionBytes ({human(cfg.keep_max_session)})') + return why, look + # The session: asked last, so that the answer is as late as possible. + look.refusal = session_alive(cfg, facts, path, look) + if look.refusal is not None: + return why, look + look.keep_first = [(name, size) for name, size, _ in found.keep] + look.keep_named = [(name, size) for name, size, _ in found.listed] + first = f'keep first: {files_text(len(found.keep))} ({human(total)})' if found.keep else 'nothing to keep first' + if found.listed: + first += f', {len(found.listed)} larger output{"s" if len(found.listed) != 1 else ""} named only' + return f'{why}; {first}', look + + +def plan_scratch(cfg, facts, plan, measure): + if not cfg.scratch_enabled: + return + projects, problem = _entries(cfg.scratch_root) + if problem: + plan.kept.append(Kept(cfg.scratch_root, SCRATCH, problem.code, problem.detail)) + for project in projects or (): + folder = os.path.join(cfg.scratch_root, project) + try: + st = os.lstat(folder) + except OSError: + continue + if stat.S_ISLNK(st.st_mode): + plan.kept.append(Kept(folder, SCRATCH, 'symbolic link', 'a link is never followed and never removed')) + continue + if not stat.S_ISDIR(st.st_mode): + continue + names, problem = _entries(folder) + if problem: + plan.kept.append(Kept(folder, SCRATCH, problem.code, problem.detail)) + for name in names or (): + path = os.path.join(folder, name) + answer = scratch_answer(cfg, facts, path) + if answer is None: + continue + why, look = answer + if look.refusal is not None: + plan.kept.append(Kept(path, SCRATCH, look.refusal.code, look.refusal.detail, guard.measure(path) if measure else None)) + else: + plan.items.append(Item('session', path, SCRATCH, why, look.size, look)) + + +def finish_scratch(cfg, facts, entry): + """An agent-scratch folder an earlier run began to delete (its journal line): an Item to finish it, or a Kept + with the reason it stays. The whole folder was judged deletable then; what is left is judged again now.""" + path = entry['path'] + started = _seconds(entry.get('at')) + if started is None or not cfg.scratch_enabled or not cfg.session_parent(os.path.dirname(path)): + return None + answer = scratch_answer(cfg, facts, path, started) + if answer is None: + return None + why, look = answer + if look.refusal is not None: + return Kept(path, SCRATCH, look.refusal.code, look.refusal.detail) + item = Item('session', path, SCRATCH, why, look.size, look) + item.started = started + return item + + # --- deleting ----------------------------------------------------------------------------------------------------- # Every delete goes through descriptors, one name at a time, and never through a link: a name that is a link now is # removed as a link, and a folder is entered only when it is the folder the name showed a moment before. @@ -142,7 +693,10 @@ def _open_parent(path): def _clear(top, names, look, shown): """Removes `names` from the open folder `top`, each folder with everything in it. Every entry must be one the last check saw (its inode, on its device): a folder moved in since, a new mount or a new file stops the delete - there, so nothing the check did not judge is removed.""" + there, so nothing the check did not judge is removed. With `look.checked` (agent scratch: when the last + liveness check began), an entry whose inode changed since then also stops it: a new file can get the inode + number of one just deleted (ext4 gives it out again at once), and only its inode time shows it is new. A file + with several hard links is let through: deleting one of its names changes the inode time of the others.""" stack = [(top, sorted(names, reverse=True), None, shown)] try: while stack: @@ -157,6 +711,10 @@ def _clear(top, names, look, shown): st = os.stat(name, dir_fd=here, follow_symlinks=False) if st.st_dev != look.ident[0] or st.st_ino not in look.seen: # seen raise Unreadable(f'{where}/{name}: not there at the last check (moved in, made or mounted since): the delete stopped') + if (look.checked is not None and st.st_ctime >= look.checked and # made since + (stat.S_ISDIR(st.st_mode) or (st.st_nlink == 1 and st.st_ino not in look.linked))): + raise Unreadable(f'{where}/{name}: changed after the last check (a new file can reuse an inode ' + 'number): the delete stopped') if not stat.S_ISDIR(st.st_mode): os.unlink(name, dir_fd=here) continue @@ -172,6 +730,22 @@ def _clear(top, names, look, shown): os.close(here) +# Linux's CLOCK_REALTIME_COARSE (Python names no constant for it): the clock the kernel stamps file times with. +CLOCK_COARSE = getattr(time, 'CLOCK_REALTIME_COARSE', 5) + + +def clock(): + """Now, by the clock file times are stamped with, so a file changed after this moment never shows an earlier + time: on Linux the coarse clock (the precise one runs up to a tick ahead of file times). Elsewhere the precise + clock, a tenth of a second back.""" + if sys.platform.startswith('linux'): + try: + return time.clock_gettime(CLOCK_COARSE) + except OSError: + pass + return time.time() - 0.1 + + def remove_tree(path, look, last=None): """Removes the file or folder at `path` if it is still what the last check judged (`look.ident`); else raises Untouched and removes nothing. A folder is emptied through its descriptor, `last` (a name in it) after @@ -585,7 +1159,8 @@ def delete_clone(item): # --- all rules ---------------------------------------------------------------------------------------------------- # The rules, by the name each one writes on what it plans: the one list of them. The disk report (report.py) needs # a class for each (config.CLASSES), and report.test.py fails for a rule here that has none. -RULES = {'worktrees': plan_worktrees, 'docker': plan_docker, 'tmp': plan_tmp, compress.RULE: compress.plan_compress} +RULES = {'worktrees': plan_worktrees, 'docker': plan_docker, 'tmp': plan_tmp, SCRATCH: plan_scratch, + compress.RULE: compress.plan_compress} # Kinds whose step never leaves a path half gone (compress.py: the original or its checked copy is always there), # so the journal of deletes in progress does not name them. WORDS: what the engine says of them (plan, run, done) # and NOUNS what it counts them as. @@ -610,6 +1185,14 @@ def recheck(cfg, facts, item): return recheck_tree(cfg, facts, item) if item.kind in ('compress', 'part'): return compress.recheck(cfg, facts, item) + if item.kind == 'session': + if not cfg.session_parent(os.path.dirname(item.path)): + return Refusal('not a candidate', 'not a session folder of the agent-scratch root of the machine file') + answer = scratch_answer(cfg, facts, item.path, getattr(item, 'started', None)) + if answer is None: + return Refusal('not a candidate', 'it changed, or the rule no longer takes it') + item.look = answer[1] + return answer[1].refusal if os.path.dirname(item.path) != cfg.tmp_root: return Refusal('not a candidate', 'not in the temp folder of the machine file') answer = tmp_answer(cfg, facts, item.path) @@ -620,5 +1203,5 @@ def recheck(cfg, facts, item): def delete(item): - {'path': delete_path, 'worktree': delete_worktree, 'clone': delete_clone, 'volume': lambda i: remove_volume(i.path), - 'compress': compress.compress, 'part': compress.remove_part}[item.kind](item) + {'path': delete_path, 'session': delete_path, 'worktree': delete_worktree, 'clone': delete_clone, + 'volume': lambda i: remove_volume(i.path), 'compress': compress.compress, 'part': compress.remove_part}[item.kind](item) diff --git a/fork-kit/cleaner/tests/cleaner_fixture.py b/fork-kit/cleaner/tests/cleaner_fixture.py index be63e7c28dd3c..22832eef9f7b0 100644 --- a/fork-kit/cleaner/tests/cleaner_fixture.py +++ b/fork-kit/cleaner/tests/cleaner_fixture.py @@ -145,6 +145,7 @@ def setUp(self): self.section = {'logFile': self.path('state', 'cleaner.log'), 'daily': {'outDir': self.path('state', 'daily')}} self.merged, self.open = [], [] self.busy, self.unit_names, self.locks, self.texts = set(), set(), set(), [] + self.words, self.transcript_times = set(), {} # process environments' words; {session id: time} self.mount_points = {'/'} self.mount_lines, self.bind_sources = [], set() # (point, device, root) and (device, root) self.mount_options = {'/': ('rw', 'relatime')} # {point: its per-mount options} @@ -217,12 +218,14 @@ def repo_cfg(self, regenerable=None, **section): return self.cfg(worktrees={'repos': [repo]}, **section) def processes(self): - return self.c.facts.Processes(busy=set(self.busy), locks=set(self.locks), texts=list(self.texts)) + return self.c.facts.Processes(busy=set(self.busy), locks=set(self.locks), texts=list(self.texts), + words=set(self.words)) def facts(self, cfg, **over): readers = {'processes': self.processes, 'units': lambda: set(self.unit_names), 'prs': lambda repo: {'merged': list(self.merged), 'open': list(self.open)}, - 'dangling': lambda: [], 'now': self.now, 'mounts': lambda: self.c.facts.Mounts(self.mount_points, self.mount_lines, self.bind_sources, self.mount_options)} + 'dangling': lambda: [], 'now': self.now, 'transcripts': lambda: dict(self.transcript_times), + 'mounts': lambda: self.c.facts.Mounts(self.mount_points, self.mount_lines, self.bind_sources, self.mount_options)} return self.c.facts.Facts(cfg, **{**readers, **over}) def plan(self, cfg, **over): @@ -343,10 +346,18 @@ def calls(self, state_path): return [json.loads(line) for line in f] def fake_proc(self, processes, locks='', sockets=()): - """A fake /proc: {pid: {'cwd': path, 'exe': path, 'fd': [paths], 'cmdline': [args], 'maps': [paths]}}.""" + """A fake /proc: {pid: {'cwd': path, 'exe': path, 'fd': [paths], 'cmdline': [args], 'maps': [paths], + 'environ': {name: value}, 'comm': short name, 'start': when it started (seconds since the epoch), + 'hidden': True (its open files cannot be read, as a process that is not dumpable), 'parent': its parent's + pid (1 when not given)}}.""" if os.path.isdir(self.path('proc')): + for d in os.listdir(self.path('proc')): + if os.path.isdir(self.path('proc', d, 'fd')) and not os.path.islink(self.path('proc', d, 'fd')): + os.chmod(self.path('proc', d, 'fd'), 0o700) shutil.rmtree(self.path('proc')) root = self.mkdir('proc') + boot = int(self.now) - 30 * DAY + self.file(os.path.join(root, 'stat'), f'cpu 1 2 3 4\nbtime {boot}\nprocesses 9\n') self.file(os.path.join(root, 'locks'), locks) self.file(os.path.join(root, 'net', 'unix'), 'Num RefCount Protocol Flags Type St Inode Path\n' + ''.join( f'0000000000000000: 00000002 00000000 00010000 0001 01 {i + 9} {s}\n' for i, s in enumerate(sockets)) @@ -361,8 +372,19 @@ def fake_proc(self, processes, locks='', sockets=()): os.symlink(target, os.path.join(d, 'fd', str(i))) with open(os.path.join(d, 'cmdline'), 'wb') as f: f.write(b'\0'.join(a.encode() for a in p.get('cmdline', ['sleep'])) + b'\0') + if 'environ' in p: + with open(os.path.join(d, 'environ'), 'wb') as f: + f.write(b''.join(f'{k}={v}'.encode() + b'\0' for k, v in p['environ'].items())) with open(os.path.join(d, 'maps'), 'w') as f: f.write(''.join(f'7f00-7f01 r--p 00000000 08:01 {i + 5} {m}\n' for i, m in enumerate(p.get('maps', [])))) + if 'comm' in p: + self.file(os.path.join(d, 'comm'), p['comm'] + '\n') + if 'start' in p or 'parent' in p: + ticks = round((p.get('start', boot) - boot) * os.sysconf('SC_CLK_TCK')) + self.file(os.path.join(d, 'stat'), f'{pid} ({p.get("comm", "x")}) S {p.get("parent", 1)} {pid} {pid} 0 -1 4194560 ' + + ' '.join(['0'] * 12) + f' {ticks} 1000 200\n') + if p.get('hidden'): + os.chmod(os.path.join(d, 'fd'), 0) return root diff --git a/fork-kit/cleaner/tests/compress.test.py b/fork-kit/cleaner/tests/compress.test.py index c3ecf593ff661..c4adfcd47f981 100644 --- a/fork-kit/cleaner/tests/compress.test.py +++ b/fork-kit/cleaner/tests/compress.test.py @@ -734,6 +734,24 @@ def test_folders_it_refuses(self): with self.assertRaises(machine_file.MachineFileError): self.cfg(compressCold={'keepFreeGB': 0}) + def test_never_where_agent_scratch_deletes_or_keeps_first(self): + """While agent-scratch is on, its root is a folder another rule deletes from and its keep-first archive is + one of the cleaner's own files: compress-cold takes neither, nor a folder inside or above one. Off, both may + be named.""" + scratch = self.mkdir('scratch') + archive = self.path('data', 'kept') + on = {'enabled': True, 'root': scratch, 'keepFirst': {'archive': archive}} + for path, why in ((scratch, 'another rule deletes from'), (os.path.join(scratch, 'project'), 'another rule deletes from'), + (archive, 'agentScratch.keepFirst.archive'), (os.path.join(archive, 'x'), 'agentScratch.keepFirst.archive'), + (self.path('data'), 'agentScratch.keepFirst.archive')): + with self.subTest(path=path), self.assertRaises(machine_file.MachineFileError) as raised: + self.cfg(compressCold={'folders': [{'path': path, 'days': 30}]}, agentScratch=on) + self.assertIn(why, str(raised.exception)) + off = {**on, 'enabled': False} + for path in (scratch, archive): + cfg = self.cfg(compressCold={'folders': [{'path': path, 'days': 30}]}, agentScratch=off) + self.assertEqual([f.path for f in cfg.cold_folders], [path]) + def test_a_claude_home_that_is_a_link_is_never_taken_where_it_leads(self): real = self.mkdir('data', 'claude-real', 'projects') os.symlink(os.path.dirname(real), os.path.join(self.home, '.claude')) @@ -877,6 +895,12 @@ def test_folders_inside_must_keep_paths_may_be_named(self): 'config: no overlap with another rule': ('config', "raise machine_file.MachineFileError(f'{at}.path: {text} holds or is inside {root}, a folder '\n" " 'another rule deletes from')", 'pass', Config, 'test_folders_it_refuses'), + 'config: never the agent-scratch root': ('config', "for root in self.roots() | ({self.scratch_root} if self.scratch_enabled else set()):\n if _holds(path, root)", + "for root in self.roots():\n if _holds(path, root)", + Config, 'test_never_where_agent_scratch_deletes_or_keeps_first'), + 'config: never the keep-first archive': ('config', "('daily.outDir', out_text, self.daily_dir)) + archive:", + "('daily.outDir', out_text, self.daily_dir)):", + Config, 'test_never_where_agent_scratch_deletes_or_keeps_first'), 'the report counts compressing': ("report", "if verb == 'compressed':", 'if False:', Report, 'test_the_report_counts_what_compressing_freed'), } diff --git a/fork-kit/cleaner/tests/engine.test.py b/fork-kit/cleaner/tests/engine.test.py index 8a5d3eb8b1b81..113bbd429a10e 100644 --- a/fork-kit/cleaner/tests/engine.test.py +++ b/fork-kit/cleaner/tests/engine.test.py @@ -794,7 +794,8 @@ def test_every_key_says_whether_it_can_loosen_a_refusal(self): keys = set(cf.SPEC) | {f'worktrees.{k}' for k in cf.WORKTREES_SPEC} | {f'repos.{k}' for k in cf.REPO_SPEC} \ | {f'docker.{k}' for k in cf.DOCKER_SPEC} | {f'tmp.{k}' for k in cf.TMP_SPEC} \ | {f'daily.{k}' for k in cf.DAILY_SPEC} | {f'compare.{k}' for k in cf.COMPARE_SPEC} \ - | {f'compressCold.{k}' for k in cf.COMPRESS_SPEC} | {f'folders.{k}' for k in cf.FOLDER_SPEC} + | {f'compressCold.{k}' for k in cf.COMPRESS_SPEC} | {f'folders.{k}' for k in cf.FOLDER_SPEC} \ + | {f'agentScratch.{k}' for k in cf.SCRATCH_SPEC} | {f'keepFirst.{k}' for k in cf.KEEP_FIRST_SPEC} self.assertEqual(set(cf.KEY_KINDS), keys) self.assertLessEqual(set(cf.KEY_KINDS.values()), {'own file', 'floor', 'keeps more', 'picks', 'loosens', 'compare only'}) self.assertEqual([k for k, v in cf.KEY_KINDS.items() if v == 'loosens'], ['repos.regenerable']) @@ -811,7 +812,7 @@ def test_the_launcher_runs(self): MUTATIONS = { 'tmp: no patterns, nothing': ('rules', 'if not cfg.tmp_patterns:', 'if False:', Tmp, 'test_no_patterns_means_nothing_is_looked_at'), - 'tmp: the days of the pattern': ('rules', 'if age < days * DAY:', 'if False:', Tmp, 'test_a_pattern_gives_the_days'), + 'tmp: the days of the pattern': ('rules', 'if age < days * DAY:\n return None', 'if False:\n return None', Tmp, 'test_a_pattern_gives_the_days'), 'tmp: only names a pattern matches': ('rules', 'match = None', "match = ('*', 1)", Tmp, 'test_a_pattern_gives_the_days'), 'tmp: the first pattern': ('rules', 'for pattern, days in cfg.tmp_patterns:', 'for pattern, days in reversed(cfg.tmp_patterns):', Tmp, 'test_the_first_pattern_that_matches_counts'), @@ -819,14 +820,15 @@ def test_the_launcher_runs(self): 'tmp: the inode time counts': ('guard', 'return max(st.st_mtime, st.st_ctime)', 'return st.st_mtime', Tmp, 'test_a_folder_copied_today_with_old_dates_is_not_old'), 'tmp: names it never takes': ('rules', 'if matches(name, TMP_NEVER):', 'if False:', Tmp, 'test_names_it_never_takes'), - 'tmp: the hard refusals': ('rules', "look = guard.examine(cfg, facts, path, 'path')", 'look = guard.Look(size=1, newest=0.0)', + 'tmp: the hard refusals': ('rules', "look = guard.examine(cfg, facts, path, 'path')\n age", 'look = guard.Look(size=1, newest=0.0)\n age', Tmp, 'test_the_hard_refusals_hold_here_too'), 'tmp: the last check before a delete': ('engine', 'refusal = rules.recheck(cfg, facts, item)', 'refusal = None', Tmp, 'test_everything_is_read_again_just_before_the_delete'), 'tmp: at least one day': ('config', 'if type(days) is not int or days < 1:', 'if False:', Tmp, 'test_bad_patterns_are_refused'), 'tmp: a root that is safe': ('config', 'if _holds(root, self.home):', 'if False:', Tmp, 'test_bad_patterns_are_refused'), 'docker: off by default': ('rules', 'if not cfg.docker_enabled:', 'if False:', Docker, 'test_it_is_off_unless_the_config_turns_it_on'), - 'docker: off by default in the config': ('config', "'enabled': (bool, False, None)", "'enabled': (bool, True, None)", + 'docker: off by default in the config': ('config', "'enabled': (bool, False, None),\n 'days': (int, MIN_DOCKER_DAYS", + "'enabled': (bool, True, None),\n 'days': (int, MIN_DOCKER_DAYS", Docker, 'test_it_is_off_unless_the_config_turns_it_on'), 'docker: named never': ('rules', 'if not ANONYMOUS.match(volume.name):', 'if False:', Docker, 'test_a_named_volume_is_never_removed'), 'docker: used by a container': ('facts', "'--filter', 'dangling=true'", "'--filter', 'dangling=false'", @@ -852,7 +854,7 @@ def test_the_launcher_runs(self): 'one log line per delete': ('engine', "out(f'{done} {getattr(item, \"result\", None) or told}')", 'pass', Engine, 'test_run_writes_one_log_line_for_each_delete'), 'the log is trimmed': ('engine', 'if len(lines) > self.cfg.log_lines:', 'if False:', Engine, 'test_the_log_keeps_its_newest_lines'), - 'a failed delete fails the run': ('engine', 'failed += 1', 'pass', Engine, 'test_a_delete_that_fails_is_reported_and_the_run_goes_on'), + 'a failed delete fails the run': ('engine', "failed += 1\n item.done = 'failed'\n gone", "item.done = 'failed'\n gone", Engine, 'test_a_delete_that_fails_is_reported_and_the_run_goes_on'), 'what it could not read fails the run': ('engine', "unread = unread_journal or any(k.code in ('could not read', 'half deleted') for k in plan.kept)", 'unread = False', Engine, 'test_what_it_could_not_read_makes_the_run_fail'), 'one run at a time': ('engine', 'fcntl.flock(self.lock, fcntl.LOCK_EX | fcntl.LOCK_NB)', 'pass', Engine, 'test_two_runs_at_once'), @@ -868,11 +870,11 @@ def test_the_launcher_runs(self): 'the log: a failed write stops the run': ('engine', "raise LogError(f'the log {self.cfg.log_file}: {ex.strerror}') from None", 'pass', Engine, 'test_a_log_that_cannot_be_written_stops_the_run_before_the_next_delete'), 'never as root': ('engine', 'return 2 # root', 'pass', Engine, 'test_it_refuses_to_run_as_root'), - 'the last check: read anew': ('engine', 'facts.forget()', 'pass', Engine, 'test_the_last_check_reads_the_machine_again'), + 'the last check: read anew': ('engine', 'facts.forget() # the last check reads the machine anew, not what the plan saw', 'pass', Engine, 'test_the_last_check_reads_the_machine_again'), 'the last check: could not read fails the run': ('engine', "unread = unread or any(item.refusal.code == 'could not read' for item in refused)", 'pass', Engine, 'test_what_it_could_not_read_at_the_last_check_makes_the_run_fail'), 'half deleted: noted before the delete': ('engine', 'journal.append(entry)', 'pass', Engine, 'test_a_delete_that_stopped_part_way_is_named_and_kept'), - 'half deleted: never deleted again': ('engine', "plan.items = [item for item in plan.items if item.kind == 'volume' or item.path not in halves]", 'pass', + 'half deleted: never deleted again': ('engine', "plan.items = [item for item in plan.items if item.kind == 'volume' or item.path not in halves or item in finish]", 'pass', Engine, 'test_a_run_that_was_killed_leaves_its_delete_named'), 'half deleted: fails the run': ("engine", "k.code in ('could not read', 'half deleted')", "k.code in ('could not read',)", Engine, 'test_a_delete_that_stopped_part_way_is_named_and_kept'), diff --git a/fork-kit/cleaner/tests/guard.test.py b/fork-kit/cleaner/tests/guard.test.py index 29e7d477b0dbd..637e4eaf22014 100644 --- a/fork-kit/cleaner/tests/guard.test.py +++ b/fork-kit/cleaner/tests/guard.test.py @@ -603,9 +603,9 @@ def test_a_partial_clone_is_never_made_to_fetch(self): 'in use: command text edges': ('facts', "named = re.compile(f'(?:^|(?<=[{EDGE}]))' + re.escape(path) + f'(?=$|[/{EDGE}])')", 'named = re.compile(re.escape(path))', InUse, 'test_a_path_with_spaces_inside_command_text'), 'in use: a deleted open file': ('facts', 'path = path[:-len(DELETED)]', 'pass', InUse, 'test_an_open_file_that_was_deleted_keeps_the_file_it_names'), - 'in use: the real process list': ('facts', 'paths.add(target)', 'pass', + 'in use: the real process list': ('facts', 'paths.add(target) # its folder, root or program', 'pass', InUse, 'test_a_process_using_it_keeps_it'), - 'in use: open files': ('facts', "paths.update(links(os.path.join(base, 'fd')))", 'pass', + 'in use: open files': ('facts', 'paths.add(target) # an open file', 'pass', InUse, 'test_the_real_process_list_sees_an_open_file_and_a_held_lock'), 'in use: a socket': ('facts', 'paths.add(fields[7])', 'pass', InUse, 'test_a_socket_a_process_listens_on_keeps_it'), 'in use: a held lock': ('guard', 'if held: # locks', 'if False:', InUse, 'test_a_held_lock_inside_keeps_it'), @@ -621,7 +621,7 @@ def test_a_partial_clone_is_never_made_to_fetch(self): 'KEEP file deep inside': ('guard', 'if plain and entry.name == KEEP_FILE', 'if plain and here == path and entry.name == KEEP_FILE', InUse, 'test_a_keep_file_deep_inside_keeps_the_whole_entry'), 'lock file': ('guard', 'if matches(name, LOCK_NAMES):', 'if False:', InUse, 'test_a_lock_file_is_never_deleted'), - 'directly in a configured root': ('guard', 'if os.path.dirname(path) not in cfg.roots()', 'if False', + 'directly in a configured root': ('guard', 'if not (root in cfg.roots() or cfg.session_parent(root))', 'if False', PathSafety, 'test_only_what_is_directly_in_a_configured_root'), 'a symbolic link': ('guard', 'if stat.S_ISLNK(st.st_mode):', 'if False:', PathSafety, 'test_a_symbolic_link_is_never_followed'), 'a symbolic link on the path': ('guard', 'if os.path.realpath(root) != root:', 'if False:', diff --git a/fork-kit/cleaner/tests/report.test.py b/fork-kit/cleaner/tests/report.test.py index 5cdb6087071fb..6ed7db52bd7ae 100644 --- a/fork-kit/cleaner/tests/report.test.py +++ b/fork-kit/cleaner/tests/report.test.py @@ -85,7 +85,8 @@ def test_the_classes_are_the_cleaners_own_list(self): def test_every_rule_and_every_must_keep_path_has_a_class(self): cf = self.c.config sources = [s for _, s in cf.CLASSES] - self.assertEqual(sorted(s for s in sources if isinstance(s, str)), ['rule:compress-cold', 'rule:docker', 'rule:tmp', 'rule:worktrees']) + self.assertEqual(sorted(s for s in sources if isinstance(s, str)), + ['rule:agent-scratch', 'rule:compress-cold', 'rule:docker', 'rule:tmp', 'rule:worktrees']) self.assertEqual(sorted(p for s in sources if isinstance(s, tuple) for p in s), sorted(cf.MUST_KEEP)) self.assertEqual(cf.MUST_KEEP, cf.AGENT_HOMES + cf.T3_DATA) # Every rule of the rule list (rules.RULES, which plan() runs) has its class, and every class names a rule. @@ -104,8 +105,11 @@ def test_every_rule_a_plan_runs_is_in_the_rule_list(self): os.chmod(self.tmp, 0o300) cold = self.mkdir('cold') os.chmod(cold, 0o300) + scratch = self.mkdir('scratch') + os.chmod(scratch, 0o300) cfg = self.repo_cfg(docker={'enabled': True}, tmp={'root': self.tmp, 'patterns': {'job-*': 1}}, - compressCold={'folders': [{'path': cold, 'days': 7}]}) + compressCold={'folders': [{'path': cold, 'days': 7}]}, + agentScratch={'enabled': True, 'root': scratch}) def dangling(): raise self.c.facts.Unreadable('Docker is down') diff --git a/fork-kit/cleaner/tests/scratch.test.py b/fork-kit/cleaner/tests/scratch.test.py new file mode 100644 index 0000000000000..f385a134c1c90 --- /dev/null +++ b/fork-kit/cleaner/tests/scratch.test.py @@ -0,0 +1,965 @@ +"""The `agent-scratch` rule and its keep-first step (lib/cleaner/rules.py, engine.py, config.py, facts.py): the +scratch folders agent sessions leave behind (`//`). Everything lives in the test's own +temporary folder: the scratch root, the archive, the transcripts and a fake /proc. RedCheck breaks each refusal and +each keep-first step in a copy of the code and shows its test then fails. + + python3 -I -B fork-kit/cleaner/tests/scratch.test.py +""" +import hashlib, json, os, sys, time, unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cleaner_fixture as fx # noqa: E402 + +import machine_file # noqa: E402 + +DAY = fx.DAY +PROJECT = '-work-acme-widget' +SID = '7d2f9c41-58ab-4e0f-9a3c-2b6e1f0d8c57' +OTHER = '11111111-2222-4333-8444-555555555555' +REPORT = '# verdict\nall good\n' + + +class Case(fx.Case): + c = fx.load() + + def setUp(self): + super().setUp() + self.scratch = self.mkdir('scratch') + self.archive = self.path('state', 'kept') + + def scfg(self, keep=None, **scratch): + section = {'enabled': True, 'root': self.scratch, 'keepFirst': {'archive': self.archive, **(keep or {})}, **scratch} + return self.cfg(agentScratch=section) + + def session(self, sid=SID, days=10, files=None, project=PROJECT): + """A session folder `days` days old. `files`: {relative path: text}; by default a small report, a result, + a note one folder down, and things not worth keeping.""" + path = self.mkdir('scratch', project, sid) + files = files if files is not None else { + 'scratchpad/notes.md': REPORT, 'scratchpad/result.json': '{"ok": true}\n', + 'scratchpad/run-1/summary.md': 'summary\n', 'scratchpad/clone/package.json': '{}\n', + 'scratchpad/clone/src/deep.md': 'deep\n', 'scratchpad/shot.png': 'png', 'tasks/b1.output': 'output\n'} + for rel, text in files.items(): + self.file(os.path.join(path, *rel.split('/')), text) + return self.age(path, days) + + def kept_copy(self, sid=SID, project=PROJECT, name=None): + folder = os.path.join(self.archive, project, name or sid) + with open(os.path.join(folder, 'manifest.json')) as f: + return folder, json.load(f) + + def journal(self, cfg): + return self.c.engine.read_journal(cfg) + + def boom(self, text='/proc: Permission denied'): + def read(): + raise self.c.facts.Unreadable(text) + return read + + +class Rule(Case): + def test_a_gone_idle_session_is_deleted_after_its_reports_are_kept(self): + path = self.session() + cfg = self.scfg() + plan = self.plan(cfg) + item = self.assertDeletes(plan, path, 'agent-scratch') + self.assertIn('session gone, not changed for 10 days; keep first: 4 files', item.why) + self.assertEqual([name for name, _ in item.look.keep_first], + ['scratchpad/notes.md', 'scratchpad/result.json', 'scratchpad/run-1/summary.md', 'tasks/b1.output']) + code, lines = self.run_cleaner(cfg, really=False) # the plan shows both + self.assertEqual(code, 0, lines) + self.assertIn(f' would keep first {path}/scratchpad/notes.md (19 B)', lines) + self.assertTrue(any(line.startswith(f'would delete {path} (') for line in lines), lines) + self.assertTrue(any(line.startswith(' agent-scratch: would keep first 4 files') for line in lines), lines) + self.assertFalse(os.path.exists(self.archive)) + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + self.assertFalse(os.path.lexists(path)) + self.assertTrue(os.path.isdir(os.path.join(self.scratch, PROJECT)), 'a project folder never goes') + folder, manifest = self.kept_copy() + self.assertEqual((manifest['source'], manifest['project'], manifest['session'], manifest['rule']), + (path, PROJECT, SID, 'agent-scratch')) + self.assertEqual([f['path'] for f in manifest['files']], + ['scratchpad/notes.md', 'scratchpad/result.json', 'scratchpad/run-1/summary.md', 'tasks/b1.output']) + self.assertEqual((manifest['notCopied'], manifest['linksOut']), ([], [])) + for f in manifest['files']: + with open(os.path.join(folder, *f['path'].split('/')), 'rb') as copy: + body = copy.read() + self.assertEqual((hashlib.sha256(body).hexdigest(), len(body)), (f['sha256'], f['bytes'])) + with open(os.path.join(folder, 'scratchpad', 'notes.md')) as f: + self.assertEqual(f.read(), REPORT) + with open(cfg.log_file) as f: + log = f.read().splitlines() + first = [line for line in log if 'kept first ' in line] + self.assertEqual(len(first), 1, log) + self.assertLess(log.index(first[0]), next(i for i, line in enumerate(log) if ' deleting ' in line)) + freed = self.c.report.freed(cfg) + self.assertEqual((freed.deletes, freed.skipped), (1, 0), 'the report reads the keep-first line as the cleaner\'s own') + self.assertEqual(self.journal(cfg), []) + code, lines = self.run_cleaner(cfg) # runs again: nothing left to do + self.assertEqual((code, [line for line in lines if line.startswith(('deleted ', 'kept first '))]), (0, [])) + + def test_a_session_with_nothing_worth_keeping_is_deleted_without_a_copy(self): + path = self.session(files={'scratchpad/clone/package.json': '{}', 'scratchpad/a.png': 'png'}) + cfg = self.scfg() + self.assertIn('nothing to keep first', self.assertDeletes(self.plan(cfg), path).why) + self.assertEqual(self.run_cleaner(cfg)[0], 0) + self.assertFalse(os.path.lexists(path)) + self.assertFalse(os.path.exists(os.path.join(self.archive, PROJECT, SID))) + + def test_patterns_from_the_machine_file_pick_what_is_kept_first(self): + path = self.session() + plan = self.plan(self.scfg({'patterns': ['scratchpad/*.txt', 'scratchpad/run-*/*.md']})) + self.assertEqual([n for n, _ in self.assertDeletes(plan, path).look.keep_first], + ['scratchpad/run-1/summary.md', 'tasks/b1.output']) + plan = self.plan(self.scfg({'patterns': ['scratchpad/*.txt'], 'outputs': ['tasks/none-*']})) + self.assertIn('nothing to keep first', self.assertDeletes(plan, path).why) + for empty in ({'patterns': []}, {'outputs': []}): # keep first cannot be turned off + with self.subTest(empty=empty), self.assertRaises(machine_file.MachineFileError): + self.scfg(empty) + + def test_a_report_too_large_to_keep_keeps_the_session(self): + path = self.session(files={'scratchpad/notes.md': REPORT, 'scratchpad/dump.json': 'x' * (300 * 1024)}) + cfg = self.scfg() + self.assertKept(self.plan(cfg), path, 'report too large to keep first', 'scratchpad/dump.json') + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + self.assertTrue(os.path.isfile(os.path.join(path, 'scratchpad', 'dump.json'))) + self.assertFalse(os.path.exists(self.archive), 'nothing is copied for a session that stays') + item = self.c.rules.Item('session', path, 'agent-scratch', '', 1, self.c.guard.examine(cfg, self.facts(cfg), path, 'path')) + with self.assertRaises(self.c.rules.KeepFirstError): # nor by keep first itself + self.c.rules.keep_first(cfg, item) + + def test_task_outputs_are_kept_first_and_a_larger_one_is_named(self): + transcript = self.file(self.path('home', '.claude', 'projects', PROJECT, SID, 'subagents', 'a.jsonl'), 'agent') + path = self.session(files={'scratchpad/notes.md': REPORT, 'tasks/b1.output': 'output\n', + 'tasks/big.output': 'y' * (300 * 1024), 'scratchpad/run.log': 'linked\n'}) + os.symlink('../scratchpad/run.log', os.path.join(path, 'tasks', 'in.output')) + os.symlink(transcript, os.path.join(path, 'tasks', 'out.output')) + os.symlink('/etc/hostname', os.path.join(path, 'tasks', 'etc.output')) + self.age(path, 10) + cfg = self.scfg() + item = self.assertDeletes(self.plan(cfg), path) + self.assertEqual([n for n, _ in item.look.keep_first], ['scratchpad/notes.md', 'tasks/b1.output', 'tasks/in.output']) + self.assertEqual(item.look.keep_named, [('tasks/big.output', 300 * 1024)]) + self.assertIn('1 larger output named only', item.why) + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + self.assertFalse(os.path.lexists(path)) + folder, manifest = self.kept_copy() + self.assertEqual([f['path'] for f in manifest['files']], ['scratchpad/notes.md', 'tasks/b1.output', 'tasks/in.output']) + with open(os.path.join(folder, 'tasks', 'in.output')) as f: + self.assertEqual(f.read(), 'linked\n') + self.assertFalse(os.path.islink(os.path.join(folder, 'tasks', 'in.output'))) + self.assertEqual([(f['path'], f['bytes'], f['sha256']) for f in manifest['notCopied']], + [('tasks/big.output', 300 * 1024, hashlib.sha256(b'y' * (300 * 1024)).hexdigest())]) + self.assertFalse(os.path.exists(os.path.join(folder, 'tasks', 'big.output'))) + self.assertEqual(sorted((f['path'], f['to']) for f in manifest['linksOut']), + [('tasks/etc.output', '/etc/hostname'), ('tasks/out.output', transcript)]) + with open(transcript) as f: + self.assertEqual(f.read(), 'agent', 'a link out is never followed') + + def test_too_much_to_keep_first_keeps_the_session_whole(self): + path = self.session() + self.assertKept(self.plan(self.scfg({'maxSessionBytes': 10})), path, 'too much to keep first', '4 files') + self.assertEqual(self.run_cleaner(self.scfg({'maxSessionBytes': 10}))[0], 0) + self.assertTrue(os.path.isdir(path)) + + def test_a_link_is_never_followed_into_or_kept_from(self): + path = self.session(files={'scratchpad/notes.md': REPORT}) + secret = self.file(self.path('home', 'private.md'), 'private') + os.symlink(secret, os.path.join(path, 'scratchpad', 'linked.md')) + self.age(path, 10) + item = self.assertDeletes(self.plan(self.scfg()), path) + self.assertEqual([n for n, _ in item.look.keep_first], ['scratchpad/notes.md']) + + def test_only_session_folders_are_candidates(self): + self.session() + other = [self.age(self.mkdir('scratch', PROJECT, 'not-a-session'), 30), + self.file(self.path('scratch', PROJECT, OTHER + '.txt'), days=30), + self.age(self.mkdir('scratch', OTHER), 30)] # directly in the root: no project folder + plan = self.plan(self.scfg()) + for path in other: + self.assertNotCandidate(plan, path) + self.assertNotCandidate(plan, os.path.join(self.scratch, PROJECT)) + os.symlink(self.path('home'), self.path('scratch', 'linked-project')) + self.assertKept(self.plan(self.scfg()), self.path('scratch', 'linked-project'), 'symbolic link') + + +class Refusals(Case): + def test_a_recent_change_keeps_it(self): + path = self.session(days=10) + self.file(os.path.join(path, 'scratchpad', 'new.md'), days=1) + self.assertNotCandidate(self.plan(self.scfg()), path) + self.age(path, 4) + self.assertNotCandidate(self.plan(self.scfg(days=5)), path) + self.assertDeletes(self.plan(self.scfg()), path) + self.assertDeletes(self.plan(self.scfg(days=3)), path) + + def test_a_live_session_is_kept_by_its_command_line(self): + path = self.session() + for text in (f'claude --session-id={SID} --model x', f'claude --resume {SID}', f'node cli.js -r "{SID}"'): + with self.subTest(text=text): + self.texts = [text] + self.assertKept(self.plan(self.scfg()), path, 'session alive') + self.texts = [f'claude --session-id={SID}0', f'claude --session-id={OTHER}'] # no whole word: not this one + self.assertDeletes(self.plan(self.scfg()), path) + + def test_a_live_session_is_kept_by_a_child_process_environment(self): + path = self.session() + root = self.fake_proc({100: {'cwd': '/'}, 200: {'cmdline': ['bash'], 'environ': {'CLAUDE_CODE_SESSION_ID': SID}}}) + live = {'processes': lambda: self.c.facts.read_processes(root, 100)} + self.assertKept(self.plan(self.scfg(), **live), path, 'session alive') + root = self.fake_proc({100: {'cwd': '/'}, 200: {'cmdline': ['bash'], 'environ': {'CLAUDE_CODE_SESSION_ID': OTHER}}}) + self.assertDeletes(self.plan(self.scfg(), **live), path) + + def test_a_process_list_that_cannot_be_read_keeps_it(self): + path = self.session() + self.assertKept(self.plan(self.scfg(), processes=self.boom()), path, 'could not read', 'process list') + code, lines = self.run_cleaner(self.scfg(), processes=self.boom()) + self.assertEqual(code, 1, lines) + self.assertTrue(os.path.isdir(path)) + + def test_a_recent_transcript_keeps_it(self): + path = self.session() + self.transcript_times = {SID: self.now - 1 * DAY} + self.assertKept(self.plan(self.scfg()), path, 'transcript changed under 3 days ago') + self.transcript_times = {SID: self.now - 5 * DAY, OTHER: self.now} + self.assertDeletes(self.plan(self.scfg()), path) + self.assertKept(self.plan(self.scfg(), transcripts=self.boom('~/.claude/projects: denied')), path, + 'could not read', 'transcripts') + + def test_transcripts_are_read_by_their_times_only(self): + home = self.path('home') + globs = [os.path.join(home, '.claude*', 'projects', '*')] + a = self.file(os.path.join(home, '.claude', 'projects', PROJECT, SID + '.jsonl'), '{}', days=4) + self.mkdir('home', '.claude-second', 'projects', PROJECT, OTHER, 'subagents') + self.file(os.path.join(home, '.claude', 'projects', PROJECT, 'notes.txt'), days=1) + third = '33333333-2222-4333-8444-555555555555' # a config folder that is a link + self.file(self.path('elsewhere', 'cfg', 'projects', PROJECT, third + '.jsonl'), '{}') + os.symlink(self.path('elsewhere', 'cfg'), os.path.join(home, '.claude-linked')) + found = self.c.facts.read_transcripts(globs) + self.assertAlmostEqual(found[SID], max(os.stat(a).st_mtime, os.stat(a).st_ctime), places=3) + self.assertIn(OTHER, found) + self.assertIn(third, found) + for missing in ([self.path('nobody', 'projects', '*')], [os.path.join(home, '.config', 'claude', 'projects', '*')]): + with self.subTest(missing=missing), self.assertRaises(self.c.facts.Unreadable): # moved: no answer + self.c.facts.read_transcripts(missing) + if os.geteuid() != 0: + fx.read_only(os.path.join(home, '.claude', 'projects', PROJECT)) + os.chmod(os.path.join(home, '.claude', 'projects', PROJECT), 0) + with self.assertRaises(self.c.facts.Unreadable): + self.c.facts.read_transcripts(globs) + + def test_the_transcript_folders_come_from_the_machine_file(self): + path = self.session() + self.assertEqual(self.scfg().transcript_globs, (os.path.join(self.home, '.claude*', 'projects', '*'),)) + real = {'transcripts': None} # the kit's own reader + self.file(os.path.join(self.home, '.claude', 'projects', PROJECT, SID + '.jsonl'), '{}', days=1) + self.assertKept(self.plan(self.scfg(), **real), path, 'transcript changed under 3 days ago') + moved = self.mkdir('moved', 'claude', 'projects', PROJECT) + self.file(os.path.join(moved, OTHER + '.jsonl'), '{}', days=1) + cfg = self.scfg(transcripts=[self.path('moved', '*', 'projects', '*')]) + self.assertDeletes(self.plan(cfg, **real), path) + cfg = self.scfg(transcripts=[self.path('moved', '*', 'projects', '*'), self.path('gone', 'projects', '*')]) + self.assertKept(self.plan(cfg, **real), path, 'could not read', 'no folder matches') + code, lines = self.run_cleaner(cfg, **real) + self.assertEqual(code, 1, lines) + self.assertTrue(os.path.isdir(path)) + + def test_a_lock_file_keeps_it(self): + for name in ('session.lock', 'lock'): + with self.subTest(name=name): + path = self.session(files={'scratchpad/notes.md': REPORT, name: ''}) + self.assertKept(self.plan(self.scfg()), path, 'lock file', name) + os.unlink(os.path.join(path, name)) + path = self.session(files={'scratchpad/notes.md': REPORT, 'scratchpad/clone/yarn.lock': ''}) + self.assertDeletes(self.plan(self.scfg()), path) # deeper: a clone's own, not the session's + + def live(self, processes): + root = self.fake_proc({100: {'cwd': '/'}, **processes}) + return {'processes': lambda: self.c.facts.read_processes(root, 100)} + + def test_a_process_with_a_handle_or_its_folder_in_it_keeps_it(self): + path = self.session() + cfg = self.scfg() + for p in ({'fd': [os.path.join(path, 'tasks')]}, {'cwd': os.path.join(path, 'scratchpad', 'clone')}, + {'fd': [os.path.join(path, 'tasks', 'b1.output') + ' (deleted)']}): + with self.subTest(p=p): + # An absolute command line, as real agents have: a relative one is read under the working folder. + live = self.live({200: {'cmdline': ['/opt/agent/bin/claude'], 'start': self.now, **p}}) + self.assertKept(self.plan(cfg, **live), path, 'in use') # the plan: the guard + refusal = self.c.rules.session_alive(cfg, self.facts(cfg, **live), path, None) # the last word + self.assertEqual((refusal.code, refusal.detail), + ('session alive', 'a process works in it, has a file or folder in it open, or names its path')) + live = self.live({200: {'cmdline': ['/opt/agent/bin/claude'], 'start': self.now, 'fd': [os.path.join(self.scratch, PROJECT, SID + '0')]}}) + self.assertDeletes(self.plan(cfg, **live), path) + + def test_a_session_cleared_into_a_new_id_is_kept_by_its_process(self): + path = self.session() + cfg = self.scfg() + on = os.path.join(self.scratch, PROJECT, OTHER, 'tasks') # /clear: the same process, a new session id + for start in (None, self.now - 20 * DAY): + with self.subTest(start=start): + p = {'cmdline': ['claude'], 'fd': [on], 'comm': 'claude'} + live = self.live({200: p if start is None else {**p, 'start': start}}) + self.assertKept(self.plan(cfg, **live), path, 'session alive', f'works in {OTHER} of the same project') + for p in ({'start': self.now - 1 * DAY}, # began after this one was left + {'start': self.now - 20 * DAY, 'environ': {'CLAUDE_CODE_SESSION_ID': OTHER}}, # says it is that one + {'start': self.now - 20 * DAY, 'fd': [os.path.join(self.scratch, '-work-acme-other', OTHER, 'tasks')]}): + with self.subTest(p=p): + live = self.live({200: {'cmdline': ['claude'], 'fd': [on], **p}}) + self.assertDeletes(self.plan(cfg, **live), path) + + @unittest.skipIf(os.geteuid() == 0, 'root reads every process') + def test_a_process_of_the_user_that_cannot_be_read_keeps_it(self): + path = self.session() + cfg = self.scfg() + live = self.live({200: {'comm': 'node', 'hidden': True}}) + self.assertKept(self.plan(cfg, **live), path, 'could not read', 'process 200 (node)') + for name in ('gpg-agent', 'systemd'): # helpers that hide on purpose + with self.subTest(name=name): + self.assertDeletes(self.plan(cfg, **self.live({200: {'comm': name, 'hidden': True}})), path) + + @unittest.skipIf(os.geteuid() == 0, 'root reads every process') + def test_a_hidden_process_is_judged_by_its_nearest_readable_ancestor(self): + path = self.session() + cfg = self.scfg() + codex = {'comm': 'codex', 'hidden': True, 'cmdline': ['codex', 'exec']} + shell = {'comm': 'bash', 'cmdline': ['/bin/bash'], 'start': self.now - 20 * DAY} + # The user's service manager, even one that can be read: what it starts is no agent's. + manager = {'comm': 'systemd', 'cmdline': ['/lib/systemd/systemd', '--user'], 'parent': 1} + cases = ( + # a Codex started by another session's tool: that session's, not this one's + ('other session', {300: {**shell, 'environ': {'CLAUDE_CODE_SESSION_ID': OTHER}}, 200: {**codex, 'parent': 300}}, None), + ('a server above it', {300: {**shell, 'cmdline': ['/usr/bin/node', 'server.js']}, 200: {**codex, 'parent': 300}}, None), + ('through a hidden parent', {300: shell, 301: {**codex, 'parent': 300}, 200: {**codex, 'parent': 301}}, None), + # the ancestor decides for it: its environment, its working folder + ('this session', {300: {**shell, 'environ': {'CLAUDE_CODE_SESSION_ID': SID}}, 200: {**codex, 'parent': 300}}, + 'session alive'), + ('works in it', {300: {**shell, 'cwd': os.path.join(path, 'scratchpad')}, 200: {**codex, 'parent': 300}}, + 'in use'), + # no readable ancestor short of pid 1 or the user's service manager: nothing can be said + ('under pid 1', {200: {**codex, 'parent': 1}}, 'could not read'), + ('under the service manager', {250: manager, 200: {**codex, 'parent': 250}}, 'could not read'), + ('under a hidden service manager', {250: {**manager, 'hidden': True}, 200: {**codex, 'parent': 250}}, 'could not read'), + ('its parent ended', {200: {**codex, 'parent': 4321}}, 'could not read'), + ('a parent that is its own child', {200: {**codex, 'parent': 201}, 201: {**codex, 'parent': 200}}, 'could not read'), + ) + for name, processes, code in cases: + with self.subTest(case=name): + plan = self.plan(cfg, **self.live(processes)) + if code is None: + self.assertDeletes(plan, path) + else: + self.assertKept(plan, path, code) + + @unittest.skipIf(os.geteuid() == 0, 'root reads every process') + def test_a_helper_name_excuses_only_what_cannot_be_read(self): + path = self.session() + cfg = self.scfg() + for p in ({'cwd': os.path.join(path, 'scratchpad'), 'cmdline': ['/usr/bin/gpg-agent', '--daemon']}, + {'cmdline': ['gpg-agent', '--homedir', os.path.join(path, 'scratchpad', 'gnupg'), '--daemon']}): + with self.subTest(p=p): + live = self.live({200: {'comm': 'gpg-agent', 'hidden': True, **p}}) + refusal = self.c.rules.session_alive(cfg, self.facts(cfg, **live), path, None) + self.assertEqual(refusal.code if refusal is not None else None, 'session alive') + + def test_a_credential_name_keeps_it(self): + path = self.session(files={'scratchpad/notes.md': REPORT, 'scratchpad/app/.env': 'A=1'}) + self.assertKept(self.plan(self.scfg()), path, 'credential name') + + def test_git_work_that_exists_only_here_is_kept_and_listed(self): + path = self.session(files={'scratchpad/notes.md': REPORT}) + clone = self.clone('w1', into=os.path.join(path, 'scratchpad')) + self.age(path, 10) + self.assertDeletes(self.plan(self.scfg()), path) # clean and pushed: may go + self.file(os.path.join(clone, 'draft.txt'), days=10) + self.age(path, 10) + self.assertKept(self.plan(self.scfg()), path, 'git: uncommitted or untracked', 'scratchpad/w1') + self.commit(clone, 'draft.txt') + self.age(path, 10) + self.assertKept(self.plan(self.scfg()), path, 'git: unpushed commits', 'scratchpad/w1') + code, lines = self.run_cleaner(self.scfg()) + self.assertEqual(code, 0, lines) + self.assertIn(f'kept {path} (agent-scratch: git: unpushed commits: scratchpad/w1', '\n'.join(lines)) + self.assertTrue(os.path.isdir(clone)) + + def test_the_session_comes_back_while_keep_first_copies(self): + path = self.session() + cfg = self.scfg() + copy = self.c.rules.keep_first + + def slow(cfg, item): + told = copy(cfg, item) + self.texts = [f'claude --resume {SID}'] # it comes back during the copy + return told + with mock.patch.object(self.c.rules, 'keep_first', slow): + code, lines = self.run_cleaner(cfg) + self.assertEqual([line for line in lines if line.startswith('REFUSED')], + [f'REFUSED {path} (agent-scratch: session alive: a process names its id (command line or environment))'], lines) + self.assertTrue(os.path.isfile(os.path.join(path, 'scratchpad', 'notes.md'))) + self.assertEqual(self.kept_copy()[1]['session'], SID, 'the copy stays; the next run uses it again') + self.assertEqual(self.journal(cfg), []) + + def test_the_session_comes_back_between_plan_and_delete(self): + path = self.session() + cfg = self.scfg() + facts = self.facts(cfg) + plan = self.c.rules.plan(cfg, facts) + self.assertDeletes(plan, path) + self.texts = [f'claude --resume {SID}'] # it comes back + failed, lines = self.delete_all(cfg, facts, plan.items) + self.assertEqual(failed, 0) + self.assertEqual([line for line in lines if line.startswith('REFUSED')], + [f'REFUSED {path} (agent-scratch: session alive: a process names its id (command line or environment))']) + self.assertTrue(os.path.isfile(os.path.join(path, 'scratchpad', 'notes.md'))) + self.assertFalse(os.path.exists(self.archive), 'nothing is copied for a session that stays') + self.texts = [] + self.file(os.path.join(path, 'scratchpad', 'later.md')) # or writes again + failed, lines = self.delete_all(cfg, facts, plan.items) + self.assertTrue(any(line.startswith(f'REFUSED {path} (agent-scratch: not a candidate') for line in lines), lines) + self.assertTrue(os.path.isdir(path)) + + +class KeepFirst(Case): + def test_the_copy_is_verified_before_anything_is_deleted(self): + path = self.session() + cfg = self.scfg() + with mock.patch.object(self.c.rules, '_verified', return_value=False): + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'FAILED {path} (agent-scratch: keep first: ') and 'nothing deleted' in line + for line in lines), lines) + self.assertTrue(os.path.isfile(os.path.join(path, 'scratchpad', 'notes.md'))) + self.assertFalse(os.path.exists(os.path.join(self.archive, PROJECT, SID)), 'an unverified copy is never the kept one') + self.assertEqual(self.journal(cfg), []) + code, lines = self.run_cleaner(cfg) # the next run makes it again + self.assertEqual(code, 0, lines) + self.assertFalse(os.path.lexists(path)) + self.assertEqual(self.kept_copy()[1]['session'], SID) + + def test_a_copy_that_cannot_be_written_keeps_the_folder(self): + path = self.session() + self.file(self.archive, 'not a folder') + cfg = self.scfg() + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'FAILED {path} (agent-scratch: keep first: ') for line in lines), lines) + self.assertTrue(os.path.isdir(path)) + self.assertEqual(self.journal(cfg), []) + + def test_a_copy_made_by_a_run_that_stopped_before_the_delete_is_used_again(self): + path = self.session() + cfg = self.scfg() + item = self.assertDeletes(self.plan(cfg), path) + self.assertIn('(verified)', self.c.rules.keep_first(cfg, item)) # the run stops here + folder, manifest = self.kept_copy() + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + self.assertTrue(any(line.startswith(f'kept first {path}: 4 files') and f'already in {folder} (verified)' in line + for line in lines), lines) + self.assertEqual(self.kept_copy()[1], manifest) + self.assertEqual(sorted(os.listdir(os.path.join(self.archive, PROJECT))), [SID]) + self.assertFalse(os.path.lexists(path)) + + def test_a_half_written_copy_is_thrown_away_and_made_again(self): + path = self.session() + partial = os.path.join(self.archive, PROJECT, f'.{SID}.partial') + self.file(os.path.join(partial, 'scratchpad', 'notes.md'), 'half') # the copy stopped half-way + cfg = self.scfg() + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + self.assertFalse(os.path.lexists(partial)) + folder, manifest = self.kept_copy() + with open(os.path.join(folder, 'scratchpad', 'notes.md')) as f: + self.assertEqual(f.read(), REPORT) + self.assertFalse(os.path.lexists(path)) + + def test_a_link_at_the_half_written_copy_is_removed_as_a_link(self): + for kind in ('folder link', 'file link', 'file', 'folder with a link in it'): + with self.subTest(kind=kind): + path = self.session() + target = self.mkdir('elsewhere', kind) + precious = self.file(os.path.join(target, 'precious.md'), 'mine') + project = self.mkdir('state', 'kept', PROJECT) + partial = os.path.join(project, f'.{SID}.partial') + if kind == 'folder link': + os.symlink(target, partial) + elif kind == 'file link': + os.symlink(precious, partial) + elif kind == 'file': + self.file(partial, 'half') + else: + os.symlink(target, self.file(os.path.join(partial, 'scratchpad', 'notes.md'), 'half') + '.link') + code, lines = self.run_cleaner(self.scfg()) + self.assertEqual(code, 0, lines) + self.assertFalse(os.path.lexists(partial)) + self.assertEqual(sorted(os.listdir(target)), ['precious.md'], 'what a link leads to stays') + with open(precious) as f: + self.assertEqual(f.read(), 'mine') + self.assertFalse(os.path.lexists(path)) + self.assertEqual(self.kept_copy()[1]['session'], SID) + self.c.rules._remove_own(os.path.join(project, SID)) # the next kind starts afresh + + def test_the_archive_folders_are_this_users_real_folders_synced_before_the_delete(self): + path = self.session() + synced = [] + real = self.c.rules._sync_folder + with mock.patch.object(self.c.rules, '_sync_folder', lambda folder: (synced.append(folder), real(folder))): + code, lines = self.run_cleaner(self.scfg()) + self.assertEqual(code, 0, lines) + project = os.path.join(self.archive, PROJECT) + for folder in (os.path.dirname(self.archive), self.archive, project, os.path.join(project, f'.{SID}.partial')): + self.assertIn(folder, synced) # each new folder is in its parent for good + self.assertEqual(os.stat(self.archive).st_mode & 0o777, 0o700) + os.rename(project, self.path('state', 'moved')) + elsewhere = self.mkdir('elsewhere') + os.symlink(elsewhere, project) # a link in the archive: no copy through it + path = self.session() + code, lines = self.run_cleaner(self.scfg()) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'FAILED {path} (agent-scratch: keep first: ') and 'a link' in line + for line in lines), lines) + self.assertEqual(os.listdir(elsewhere), []) + self.assertTrue(os.path.isdir(path)) + + def test_a_folder_swapped_for_a_link_after_the_check_is_not_read_through(self): + path = self.session() + cfg = self.scfg() + item = self.assertDeletes(self.plan(cfg), path) + secret = self.mkdir('elsewhere', 'secret') + self.file(os.path.join(secret, 'notes.md'), 'not this session\'s') + os.rename(os.path.join(path, 'scratchpad'), os.path.join(path, 'was')) + os.symlink(secret, os.path.join(path, 'scratchpad')) # between the check and the copy + with self.assertRaises(self.c.rules.KeepFirstError): + self.c.rules.keep_first(cfg, item) + self.assertFalse(os.path.exists(os.path.join(self.archive, PROJECT, SID))) + + def swap_after_the_list(self, change): + """keep_first, with `change(path)` run after it listed what to copy and before it reads a file: the names + and sizes it compares are still what the last check saw, so only the reads themselves can refuse.""" + listed = self.c.rules.keep_first_files + + def then_change(cfg, path, look): + found = listed(cfg, path, look) + change(path) + return found + return mock.patch.object(self.c.rules, 'keep_first_files', then_change) + + def test_the_reads_never_go_through_a_link_and_read_only_what_the_check_saw(self): + def to_link(path): # a folder on the way swapped for a link (to a copy of itself) + os.rename(os.path.join(path, 'scratchpad'), os.path.join(path, 'was')) + os.symlink('was', os.path.join(path, 'scratchpad')) + + def new_file(path): # a file swapped for a new one of the same size + notes = os.path.join(path, 'scratchpad', 'notes.md') + self.file(notes + '.new', REPORT.upper()) + os.rename(notes + '.new', notes) + + def file_link(path): # a file swapped for a link to a file the check saw + notes = os.path.join(path, 'scratchpad', 'notes.md') + os.rename(notes, notes + '.was') + os.symlink('notes.md.was', notes) + for change in (to_link, new_file, file_link): + with self.subTest(change=change.__name__): + path = self.session() + cfg = self.scfg() + item = self.assertDeletes(self.plan(cfg), path) + with self.swap_after_the_list(change), self.assertRaises(self.c.rules.KeepFirstError): + self.c.rules.keep_first(cfg, item) + self.assertFalse(os.path.exists(os.path.join(self.archive, PROJECT, SID))) + self.c.rules._remove_own(path) + + def test_an_output_link_is_followed_only_to_a_file_inside_the_session(self): + path = self.session(files={'scratchpad/notes.md': REPORT}) + outside = self.file(self.path('elsewhere', 'b1.output'), 'not this session\'s\n') + os.symlink(outside, self.mkdir('scratch', PROJECT, SID, 'tasks') + '/b1.output') + self.age(path, 10) + cfg = self.scfg() + look = self.c.guard.examine(cfg, self.facts(cfg), path, 'path') + look.seen.add(os.stat(outside).st_ino) # even when its inode number is one the check saw (another device) + found = self.c.rules.keep_first_files(cfg, path, look) + self.assertEqual([name for name, _, _ in found.keep], ['scratchpad/notes.md']) + self.assertEqual(found.out, [('tasks/b1.output', outside)]) + + def test_keep_first_itself_refuses_a_report_too_large_to_copy(self): + path = self.session(files={'scratchpad/notes.md': REPORT, 'scratchpad/dump.json': 'x' * (300 * 1024)}) + cfg = self.scfg() + look = self.c.guard.examine(cfg, self.facts(cfg), path, 'path') + found = self.c.rules.keep_first_files(cfg, path, look) + look.keep_first = [(name, size) for name, size, _ in found.keep] # as if the plan had let it through + item = self.c.rules.Item('session', path, 'agent-scratch', '', 1, look) + with self.assertRaisesRegex(self.c.rules.KeepFirstError, 'dump.json: larger than maxFileBytes'): + self.c.rules.keep_first(cfg, item) + self.assertFalse(os.path.exists(self.archive)) + + def test_a_link_where_an_earlier_copy_is_kept_is_not_used_again(self): + path = self.session() + cfg = self.scfg() + item = self.assertDeletes(self.plan(cfg), path) + self.c.rules.keep_first(cfg, item) # a good copy... + moved = self.path('elsewhere', 'copy') + os.makedirs(os.path.dirname(moved)) + os.rename(os.path.join(self.archive, PROJECT, SID), moved) + os.symlink(moved, os.path.join(self.archive, PROJECT, SID)) # ...now behind a link in the archive + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + told = [line for line in lines if line.startswith(f'kept first {path}: ')] + self.assertEqual(len(told), 1, lines) + self.assertNotIn('already in', told[0]) + self.assertEqual(self.kept_copy(name=SID + '.2')[1]['session'], SID) + self.assertTrue(os.path.islink(os.path.join(self.archive, PROJECT, SID))) + + def test_a_different_older_copy_stays_and_the_new_one_gets_the_next_name(self): + path = self.session() + old = os.path.join(self.archive, PROJECT, SID) + self.file(os.path.join(old, 'manifest.json'), json.dumps({'files': [{'path': 'scratchpad/notes.md', 'sha256': '0' * 64}]})) + code, lines = self.run_cleaner(self.scfg()) + self.assertEqual(code, 0, lines) + self.assertEqual(sorted(os.listdir(os.path.join(self.archive, PROJECT))), [SID, SID + '.2']) + self.assertEqual(self.kept_copy(name=SID + '.2')[1]['session'], SID) + with open(os.path.join(old, 'manifest.json')) as f: + self.assertIn('0' * 64, f.read()) + + def test_the_archive_may_not_be_in_a_folder_the_cleaner_deletes_from(self): + for archive in (os.path.join(self.scratch, 'kept'), self.path('home'), '/'): + with self.subTest(archive=archive), self.assertRaises(machine_file.MachineFileError): + self.scfg({'archive': archive}) + + +class HalfDone(Case): + def stop_half_way(self, cfg, path): + """A run whose delete stops after one file: the journal names the folder.""" + def half(item): + os.unlink(os.path.join(item.path, 'scratchpad', 'shot.png')) + raise OSError(5, 'Input/output error') + with mock.patch.object(self.c.rules, 'delete', half): + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any('(half deleted: see the next plan)' in line for line in lines), lines) + self.assertEqual([e['path'] for e in self.journal(cfg)], [path]) + + def test_a_half_deleted_session_is_finished_by_the_next_run(self): + path = self.session() + cfg = self.scfg() + self.stop_half_way(cfg, path) + code, lines = self.run_cleaner(cfg, really=False) + self.assertEqual(code, 0, lines) + self.assertTrue(any(line.startswith(f'would delete {path} (') and 'finishing the delete an earlier run began' + in line for line in lines), lines) + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 0, lines) + self.assertFalse(os.path.lexists(path)) + self.assertTrue(any('already in' in line for line in lines if line.startswith('kept first')), lines) + self.assertEqual(self.journal(cfg), []) + + def test_a_half_deleted_session_that_changed_since_is_kept(self): + path = self.session() + cfg = self.scfg() + self.stop_half_way(cfg, path) + self.file(os.path.join(path, 'scratchpad', 'after.md'), days=-0.01) + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'kept {path} (agent-scratch: half deleted: ') and + 'changed since the delete began' in line for line in lines), lines) + self.assertTrue(os.path.isdir(path)) + self.assertEqual([e['path'] for e in self.journal(cfg)], [path]) + + def test_a_half_deleted_session_that_came_back_is_kept(self): + path = self.session() + cfg = self.scfg() + self.stop_half_way(cfg, path) + self.texts = [f'claude --resume {SID}'] + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'kept {path} (agent-scratch: half deleted: ') and 'session alive' in line + for line in lines), lines) + self.assertTrue(os.path.isdir(path)) + + def test_a_file_made_after_the_last_check_stops_the_delete_even_with_a_reused_inode(self): + path = self.session() + cfg = self.scfg() + alive = self.c.rules.session_alive + made = [] + + def resumed(cfg, facts, where, look): + refusal = alive(cfg, facts, where, look) + if look is not None and look.checked is not None: # the last check, right before the delete + new = self.file(os.path.join(where, 'tasks', 'new.output'), 'resumed\n') + look.seen.add(os.lstat(new).st_ino) # ext4 can give it a just-freed number + made.append(new) + return refusal + with mock.patch.object(self.c.rules, 'session_alive', resumed): + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'FAILED {path} ') and 'changed after the last check' in line + for line in lines), lines) + self.assertTrue(made and os.path.isfile(made[0]), 'the new file stays') + self.assertEqual([e['path'] for e in self.journal(cfg)], [path]) + + def test_hard_links_do_not_stop_the_delete(self): + path = self.session(files={'scratchpad/notes.md': REPORT, 'scratchpad/a.bin': 'shared'}) + os.link(os.path.join(path, 'scratchpad', 'a.bin'), os.path.join(path, 'scratchpad', 'b.bin')) + self.age(path, 10) + cfg = self.scfg() + look = self.c.guard.examine(cfg, self.facts(cfg), path, 'path') + self.assertIsNone(look.refusal) + time.sleep(0.02) # past the clock's tick + look.checked = self.c.rules.clock() + time.sleep(0.02) # the first unlink changes the other's ctime + self.c.rules.remove_tree(path, look) + self.assertFalse(os.path.lexists(path)) + + def test_with_the_rule_off_a_half_deleted_session_stays_half_deleted(self): + path = self.session() + self.stop_half_way(self.scfg(), path) + cfg = self.cfg(agentScratch={'root': self.scratch}) + code, lines = self.run_cleaner(cfg) + self.assertEqual(code, 1, lines) + self.assertTrue(any(line.startswith(f'kept {path} (agent-scratch: half deleted: since ') for line in lines), lines) + self.assertTrue(os.path.isdir(path)) + + +class Machine(Case): + def test_an_old_machine_file_without_the_keys_leaves_the_rule_off(self): + self.session() + cfg = self.cfg() + self.assertFalse(cfg.scratch_enabled) + read = AssertionError('the scratch root was read') + with mock.patch.object(self.c.rules, '_entries', side_effect=read): + plan = self.plan(cfg) + self.assertEqual((plan.items, plan.kept), ([], [])) + self.assertEqual(self.cfg(agentScratch={}).scratch_enabled, False) + self.assertTrue(os.path.isdir(self.path('scratch', PROJECT, SID))) + + def test_an_empty_or_missing_root_is_nothing_to_do(self): + cfg = self.scfg() + self.assertEqual(self.run_cleaner(cfg), (0, ['total: deleted 0 (0 B), kept 0, failed 0'])) + os.rmdir(self.scratch) + self.assertEqual(self.run_cleaner(cfg), (0, ['total: deleted 0 (0 B), kept 0, failed 0'])) + self.mkdir('scratch', PROJECT) + self.assertEqual(self.run_cleaner(cfg)[0], 0) + + def test_the_root_and_the_keys_are_checked(self): + cfg = self.cfg(agentScratch={'enabled': True, 'root': self.path('u{uid}'), 'keepFirst': {'archive': self.archive}}) + self.assertEqual(cfg.scratch_root, self.path(f'u{os.getuid()}')) + self.assertEqual(self.cfg().scratch_root, f'/tmp/claude-{os.getuid()}') + self.assertEqual(self.cfg().keep_archive, self.path('home', '.local', 'state', 'fork-cleaner', 'kept')) + linked = self.path('linked-scratch') + os.symlink(self.scratch, linked) + for bad in ({'root': self.path('x{user}')}, {'root': self.path('home', '..')}, {'root': '/'}, {'days': 2}, + {'root': '/tmp'}, {'root': '/tmp/'}, {'root': '/var/tmp'}, {'root': '/dev/shm'}, {'root': self.home}, + {'root': linked}, {'root': self.file(self.path('a-file'))}, {'transcripts': []}, + {'transcripts': ['relative/*']}, {'keepFirst': {'patterns': []}}, {'keepFirst': {'outputs': []}}, + {'keepFirst': {'patterns': ['../x.md']}}, {'keepFirst': {'patterns': ['/abs/*.md']}}, + {'keepFirst': {'patterns': ['a//b']}}, {'keepFirst': {'maxFileBytes': 1000}}, + {'keepFirst': {'maxSessionBytes': -1}}, {'enabled': 'yes'}, {'keepFirst': {'other': 1}}): + with self.subTest(bad=bad), self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': self.scratch, **bad}) + + @unittest.skipIf(os.geteuid() == 0, 'root owns the system folders') + def test_the_root_is_one_users_own_folder(self): + for other in ('/usr', '/var/log'): + with self.subTest(other=other), self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': other, 'keepFirst': {'archive': self.archive}}) + self.assertEqual(self.scfg().scratch_root, self.scratch) + self.cfg(agentScratch={'enabled': True, 'root': self.path('not-yet'), 'keepFirst': {'archive': self.archive}}) + with mock.patch('tempfile.gettempdir', return_value=self.tmp): # $TMPDIR: this user's, still shared + with self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': self.tmp, 'keepFirst': {'archive': self.archive}}) + # The same folder reached through a link in the path: lstat sees a real folder of this user, only the + # real path shows it is the shared one. + os.symlink(os.path.dirname(self.tmp), self.path('up')) + through = self.path('up', os.path.basename(self.tmp)) + with self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': through, 'keepFirst': {'archive': self.archive}}) + runtime = self.mkdir('runtime') # the user's runtime folder: shared too + for env, root in (({'XDG_RUNTIME_DIR': runtime}, runtime), ({}, f'/run/user/{os.getuid()}')): + with self.subTest(root=root), mock.patch.dict(os.environ, env): + if not env: + os.environ.pop('XDG_RUNTIME_DIR', None) + with self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': root, 'keepFirst': {'archive': self.archive}}) + + def test_the_root_never_overlaps_a_transcript_folder(self): + logs = self.mkdir('logs', 'cfg', 'projects') + self.mkdir('logs', 'cfg', 'projects', PROJECT) + os.symlink(self.path('logs', 'cfg'), self.path('linked-cfg')) + globs = [self.path('logs', '*', 'projects', '*')] + for root in (logs, # the transcripts' own folder + self.path('logs', 'cfg', 'projects', PROJECT), # one the pattern matches + self.path('logs', 'cfg', 'projects', PROJECT, 'deeper'), # inside one + self.path('logs', 'cfg'), # holds one + self.path('linked-cfg', 'projects')): # the same, through a link + with self.subTest(root=root), self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': root, 'transcripts': globs, + 'keepFirst': {'archive': self.archive}}) + with self.assertRaises(machine_file.MachineFileError): # no folder matches yet: by the real path + self.cfg(agentScratch={'enabled': True, 'root': self.path('linked-cfg', 'projects'), + 'transcripts': [self.path('logs', '*', 'projects', 'none-*')], + 'keepFirst': {'archive': self.archive}}) + os.symlink(logs, self.path('scratch-link-target')) # a pattern that leads through a link + with self.assertRaises(machine_file.MachineFileError): + self.cfg(agentScratch={'enabled': True, 'root': logs, 'transcripts': [self.path('scratch-link-*', '*')], + 'keepFirst': {'archive': self.archive}}) + self.assertEqual(self.cfg(agentScratch={'enabled': True, 'root': self.scratch, 'transcripts': globs, + 'keepFirst': {'archive': self.archive}}).scratch_root, self.scratch) + + def test_the_compare_reads_the_rule_name(self): + lines = [f'would delete /s/{SID} (1.0 KB, agent-scratch: session gone)', + f'kept /s/{OTHER} (agent-scratch: session alive: a process names its id)'] + would, kept = self.c.compare.read_plan(lines) + self.assertEqual(would, {f'/s/{SID}': ('agent-scratch', 'session gone')}) + self.assertEqual(list(kept), [f'/s/{OTHER}']) + + +MUTATIONS = { + 'session alive': ('rules', 'if processes.names(sid):', 'if False:', Refusals, 'test_a_live_session_is_kept_by_its_command_line'), + 'session alive: whole words': ('facts', 'return word in self.words', 'return any(word in w for w in self.words)', + Refusals, 'test_a_live_session_is_kept_by_its_command_line'), + 'session alive: environment': ('facts', "found = WORD.findall(f.read().decode('utf-8', 'surrogateescape'))", + 'found = []', Refusals, 'test_a_live_session_is_kept_by_a_child_process_environment'), + 'session alive: a handle or the folder in it, asked again': ('rules', ' if processes.uses(path):\n return Refusal(', + ' if False:\n return Refusal(', + Refusals, 'test_a_process_with_a_handle_or_its_folder_in_it_keeps_it'), + 'session alive: after /clear': ('rules', 'if p.start is None or p.start <= newest:', 'if False:', + Refusals, 'test_a_session_cleared_into_a_new_id_is_kept_by_its_process'), + 'session alive: after /clear, only before this one changed': ('rules', 'if p.start is None or p.start <= newest:', + 'if True:', Refusals, + 'test_a_session_cleared_into_a_new_id_is_kept_by_its_process'), + 'session alive: after /clear, not a session it names': ('rules', 'if other == sid or (SESSION_NAME.match(other) and other in p.words):', + 'if other == sid:', Refusals, + 'test_a_session_cleared_into_a_new_id_is_kept_by_its_process'), + 'session alive: a process\'s own open files': ('facts', ' paths.add(target) # an open file\n one.paths.add(target)', + ' paths.add(target) # an open file', Refusals, + 'test_a_session_cleared_into_a_new_id_is_kept_by_its_process'), + 'session alive: when a process started': ('facts', 'one.start, one.parent = _stat(base, boot)', + 'one.start, one.parent = None, _stat(base, boot)[1]', + Refusals, 'test_a_session_cleared_into_a_new_id_is_kept_by_its_process'), + 'could not read: a hidden process of the user': ('rules', " if unread:\n return Refusal('could not read'", + " if False:\n return Refusal('could not read'", + Refusals, 'test_a_process_of_the_user_that_cannot_be_read_keeps_it'), + 'could not read: helpers that hide on purpose': ('rules', 'p.hidden and p.name not in HIDDEN_OK', 'p.hidden', + Refusals, 'test_a_process_of_the_user_that_cannot_be_read_keeps_it'), + 'could not read: open files hidden': ('facts', 'fds = []\n one.hidden = one.hidden or (mine and ex.errno in (errno.EACCES, errno.EPERM))', + 'fds = []', Refusals, 'test_a_process_of_the_user_that_cannot_be_read_keeps_it'), + 'could not read: a hidden process is its readable ancestor\'s': ('rules', 'and _judged_by(p, processes.own) is None]', ']', + Refusals, 'test_a_hidden_process_is_judged_by_its_nearest_readable_ancestor'), + 'could not read: the parent from stat': ('facts', 'one.start, one.parent = _stat(base, boot)', + 'one.start, one.parent = _stat(base, boot)[0], None', + Refusals, 'test_a_hidden_process_is_judged_by_its_nearest_readable_ancestor'), + 'could not read: the walk goes past a hidden parent': ('rules', ' pid = q.parent\n return None', ' return None\n return None', + Refusals, 'test_a_hidden_process_is_judged_by_its_nearest_readable_ancestor'), + 'could not read: the walk stops at the service manager': ('rules', 'if q is None or q.name == USER_MANAGER:', 'if q is None:', + Refusals, 'test_a_hidden_process_is_judged_by_its_nearest_readable_ancestor'), + 'session alive: the working folder': ('facts', 'paths.add(target) # its folder, root or program', + "paths.add(target) if name == 'exe' else None", + Refusals, 'test_a_process_with_a_handle_or_its_folder_in_it_keeps_it'), + 'session alive: a helper\'s readable parts': ('facts', 'paths.add(target) # its folder, root or program', + "paths.add(target) if name == 'exe' else None", + Refusals, 'test_a_helper_name_excuses_only_what_cannot_be_read'), + 'session alive: asked again after keep first': ('engine', 'refusal = rules.session_alive(cfg, facts, item.path, item.look)', + 'refusal = None', Refusals, 'test_the_session_comes_back_while_keep_first_copies'), + 'transcripts: a folder that is not there is no answer': ('facts', "raise Unreadable(f'no folder matches {pattern}')", 'pass', + Refusals, 'test_the_transcript_folders_come_from_the_machine_file'), + 'lock file: in the session folder': ('rules', " look.refusal = Refusal('lock file', some(locks))\n", '', + Refusals, 'test_a_lock_file_keeps_it'), + 'transcript': ('rules', 'if said is not None and facts.now - said < days * DAY:', 'if False:', + Refusals, 'test_a_recent_transcript_keeps_it'), + 'days unchanged': ('rules', " if age < days * DAY:\n return None\n why = f'session gone", + " if False:\n return None\n why = f'session gone", Refusals, 'test_a_recent_change_keeps_it'), + 'too much to keep first': ('rules', 'if total > cfg.keep_max_session:', 'if False:', + Rule, 'test_too_much_to_keep_first_keeps_the_session_whole'), + 'report too large: the plan keeps it': ('rules', " if found.held:\n look.refusal", " if False:\n look.refusal", + Rule, 'test_a_report_too_large_to_keep_keeps_the_session'), + 'task outputs are kept first': ('rules', '((False, cfg.keep_patterns), (True, cfg.keep_outputs))', '((False, cfg.keep_patterns),)', + Rule, 'test_task_outputs_are_kept_first_and_a_larger_one_is_named'), + 'a larger output is named, not held': ('rules', ' elif outputs:\n', ' elif False:\n', + Rule, 'test_task_outputs_are_kept_first_and_a_larger_one_is_named'), + 'an output link resolved inside': ('rules', ' source, st = resolved', ' continue', + Rule, 'test_task_outputs_are_kept_first_and_a_larger_one_is_named'), + 'keep first cannot be turned off': ('config', " if not value:\n return 'must name at least one pattern (keep first", + " if False:\n return 'must name at least one pattern (keep first", + Rule, 'test_patterns_from_the_machine_file_pick_what_is_kept_first'), + 'keep first before the delete': ('engine', 'kept = rules.keep_first(cfg, item)', 'kept = None', + Rule, 'test_a_gone_idle_session_is_deleted_after_its_reports_are_kept'), + 'a failed keep first stops the delete': ('engine', "out(f'FAILED {item.label} ({item.rule}: keep first: {ex}): nothing deleted')\n continue", + "kept = None", KeepFirst, 'test_the_copy_is_verified_before_anything_is_deleted'), + 'verify the copy before it is the kept one': ('rules', 'if not _verified(partial, wanted, named):', 'if False:', + KeepFirst, 'test_the_copy_is_verified_before_anything_is_deleted'), + 'a verified copy is used again': ('rules', 'if _verified(version, wanted, named):', 'if False:', + KeepFirst, 'test_a_copy_made_by_a_run_that_stopped_before_the_delete_is_used_again'), + 'a half-written copy: a link is not followed': ('rules', 'st = os.lstat(partial)', 'st = os.stat(partial)', + KeepFirst, 'test_a_link_at_the_half_written_copy_is_removed_as_a_link'), + 'keep first: folders entered never through a link': ('rules', 'for part in parts[:-1]:\n new = os.open(part, DIR_FLAGS, dir_fd=fd)', + 'for part in parts[:-1]:\n new = os.open(part, os.O_RDONLY | os.O_DIRECTORY, dir_fd=fd)', + KeepFirst, 'test_the_reads_never_go_through_a_link_and_read_only_what_the_check_saw'), + 'keep first: each file read is the one the check saw': ('rules', 'if not stat.S_ISREG(st.st_mode) or st.st_ino not in look.seen:\n os.close(out)', + 'if not stat.S_ISREG(st.st_mode):\n os.close(out)', + KeepFirst, 'test_the_reads_never_go_through_a_link_and_read_only_what_the_check_saw'), + 'keep first: a file opened never through a link': ('rules', ' out = os.open(parts[-1], NO_FOLLOW, dir_fd=fd)', + ' out = os.open(parts[-1], os.O_RDONLY, dir_fd=fd)', + KeepFirst, 'test_the_reads_never_go_through_a_link_and_read_only_what_the_check_saw'), + 'an output link stays inside the session': ("rules", " if not target.startswith(path + '/'):\n return None", + " if False:\n return None", + KeepFirst, 'test_an_output_link_is_followed_only_to_a_file_inside_the_session'), + 'report too large: keep first refuses it itself': ('rules', " if found.held:\n raise KeepFirstError", " if False:\n raise KeepFirstError", + KeepFirst, 'test_keep_first_itself_refuses_a_report_too_large_to_copy'), + 'a reused copy is a real folder': ('rules', ' if not stat.S_ISDIR(os.lstat(folder).st_mode):\n return False', + ' if False:\n return False', + KeepFirst, 'test_a_link_where_an_earlier_copy_is_kept_is_not_used_again'), + 'keep first: what the last check saw': ('rules', " raise KeepFirstError('what is worth keeping is not what the last check saw", " pass # ('", + KeepFirst, 'test_a_folder_swapped_for_a_link_after_the_check_is_not_read_through'), + 'archive: new folders synced into their parents': ('rules', ' _sync_folder(os.path.dirname(folder))\n', '', + KeepFirst, 'test_the_archive_folders_are_this_users_real_folders_synced_before_the_delete'), + 'archive: no copy through a link': ('rules', " if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid():\n raise KeepFirstError(f'{here}", + " if False:\n raise KeepFirstError(f'{here}", + KeepFirst, 'test_the_archive_folders_are_this_users_real_folders_synced_before_the_delete'), + 'changed since the delete began': ('rules', 'if look.newest_file > started:', 'if False:', + HalfDone, 'test_a_half_deleted_session_that_changed_since_is_kept'), + 'a file changed after the last check stops the delete': ('rules', ' if (look.checked is not None and st.st_ctime >= look.checked and # made since', + ' if (False and # made since', + HalfDone, 'test_a_file_made_after_the_last_check_stops_the_delete_even_with_a_reused_inode'), + 'the last check sets when it began': ('engine', 'item.look.checked = rules.clock()', 'pass', + HalfDone, 'test_a_file_made_after_the_last_check_stops_the_delete_even_with_a_reused_inode'), + 'hard links do not stop the delete': ('rules', '(st.st_nlink == 1 and st.st_ino not in look.linked)', 'True', + HalfDone, 'test_hard_links_do_not_stop_the_delete'), + 'finish a half delete': ('engine', 'if isinstance(again, rules.Item):', 'if False:', + HalfDone, 'test_a_half_deleted_session_is_finished_by_the_next_run'), + 'one journal line per delete': ('engine', 'if journaled and not prior:\n journal.append(entry)', + 'if journaled:\n journal.append(entry)', + HalfDone, 'test_a_half_deleted_session_is_finished_by_the_next_run'), + 'report: the keep-first line is the cleaner\'s own': ('report', "'kept first ', ", '', + Rule, 'test_a_gone_idle_session_is_deleted_after_its_reports_are_kept'), + 'off unless the machine file turns it on': ('config', "'enabled': (bool, False, None),\n 'root': (str, '/tmp/claude-{uid}'", + "'enabled': (bool, True, None),\n 'root': (str, '/tmp/claude-{uid}'", + Machine, 'test_an_old_machine_file_without_the_keys_leaves_the_rule_off'), + 'root: not a shared temp folder': ('config', 'if root in shared or os.path.realpath(root) in shared:', 'if False:', + Machine, 'test_the_root_is_one_users_own_folder'), + 'root: a real folder': ('config', " if not stat.S_ISDIR(st.st_mode):\n raise machine_file.MachineFileError(f'{where}: {text} is a link", + " if False:\n raise machine_file.MachineFileError(f'{where}: {text} is a link", + Machine, 'test_the_root_and_the_keys_are_checked'), + 'root: not the user\'s runtime folder': ('config', "SHARED_TEMP + runtime + (tempfile.gettempdir(),)", + "SHARED_TEMP[:-1] + (tempfile.gettempdir(),)", Machine, 'test_the_root_is_one_users_own_folder'), + 'root: not a transcript folder': ('config', " if held is not None:\n raise machine_file.MachineFileError(f'{where}.agentScratch.root", + " if False:\n raise machine_file.MachineFileError(f'{where}.agentScratch.root", + Machine, 'test_the_root_never_overlaps_a_transcript_folder'), + 'root: a transcript folder by its real path': ('config', ' for path in (root, real):', ' for path in (root,):', + Machine, 'test_the_root_never_overlaps_a_transcript_folder'), + 'root: a transcript folder a pattern leads to': ('config', ' if os.path.isdir(folder) and (_holds(real, folder) or _holds(folder, real)):', + ' if False:', Machine, 'test_the_root_never_overlaps_a_transcript_folder'), + 'root: this user\'s own': ('config', 'if st.st_uid != os.getuid():', 'if False:', Machine, 'test_the_root_is_one_users_own_folder'), + 'the archive not where the cleaner deletes': ('config', "if self.scratch_enabled else ()", "if False else ()", + KeepFirst, 'test_the_archive_may_not_be_in_a_folder_the_cleaner_deletes_from'), +} + + +class RedCheck(unittest.TestCase): + def test_each_rule_has_a_test_that_fails_without_it(self): + fx.red_check(self, MUTATIONS) + + +if __name__ == '__main__': + unittest.main() diff --git a/fork-kit/cleaner/tests/worktrees.test.py b/fork-kit/cleaner/tests/worktrees.test.py index 30a8db2a81a6f..a9ce8741fb5e2 100644 --- a/fork-kit/cleaner/tests/worktrees.test.py +++ b/fork-kit/cleaner/tests/worktrees.test.py @@ -781,7 +781,7 @@ def test_a_list_that_may_be_cut_off_means_keep(self): 'only the listed repository': ('rules', 'if not same_repo(url, repo.github):', 'if False:', Worktrees, 'test_only_folders_of_the_listed_repository_are_looked_at'), 'the last check before a delete': ('engine', 'refusal = rules.recheck(cfg, facts, item)', 'refusal = None', Worktrees, 'test_everything_is_read_again_just_before_the_delete'), 'no --force': ('facts', "('worktree', 'remove', path)", "('worktree', 'remove', '--force', path)", Worktrees, 'test_git_removes_it_without_force'), - 'a refused git remove leaves no note': ('engine', 'if isinstance(ex, Untouched) or (', 'if (', + 'a refused git remove leaves no note': ('engine', '(isinstance(ex, Untouched) and not prior)', '(False)', Worktrees, 'test_git_removes_it_without_force'), 'regenerable: plain names only': ('config', "or any(c in v for c in '*?[]\\\\')", '', Worktrees, 'test_the_list_takes_only_plain_names'), 'regenerable: a name frees folders only': ('rules', "folders = parts if name.endswith('/') else parts[:-1]", 'folders = parts', @@ -816,7 +816,7 @@ def test_a_list_that_may_be_cut_off_means_keep(self): 'refused = True', Worktrees, 'test_a_git_remove_that_was_killed_or_failed_half_way_is_half_deleted'), 'git remove: failed to delete is not refused': ('facts', " and 'failed to delete' not in done.stderr", '', Worktrees, 'test_a_git_remove_that_was_killed_or_failed_half_way_is_half_deleted'), - 'git remove: only a refusal leaves no note': ('engine', 'if isinstance(ex, Untouched) or (', 'if isinstance(ex, Unreadable) or (', + 'git remove: only a refusal leaves no note': ('engine', '(isinstance(ex, Untouched) and not prior)', '(isinstance(ex, Unreadable) and not prior)', Worktrees, 'test_a_git_remove_that_was_killed_or_failed_half_way_is_half_deleted'), 'PR lists: read again before each delete': ('facts', 'self._prs = {}', "self._prs = getattr(self, '_prs', {})", Worktrees, 'test_the_pull_request_lists_are_read_for_the_plan_and_again_before_each_delete'),