From ca80c9fd145c923282fb38bea3ebacbcaa976806 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 20:21:03 +0000 Subject: [PATCH] Pilot dry run: fix the six review findings on #143 - Unknown spend (no host-observed cost but a spend record) stops the schedule instead of counting as zero. - The token check fails when git cannot list the tree or any file was not scanned. - ls-files -z output is split on NUL without stripping, so a name that starts with a space is scanned. - A contract without an evolution budget is left to check_budget instead of raising KeyError. - The printed receipt is the redacted one. - The exit code follows the final verdict after the credential scan. Each finding has a test in ReviewFindings. Co-Authored-By: Claude Opus 5.5 --- scripts/pilot_dry_run.py | 65 +++++++++++++++++++++++++------------ tests/test_pilot_dry_run.py | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 21 deletions(-) diff --git a/scripts/pilot_dry_run.py b/scripts/pilot_dry_run.py index 40225f35..649a348f 100644 --- a/scripts/pilot_dry_run.py +++ b/scripts/pilot_dry_run.py @@ -244,11 +244,17 @@ def tail(text: str, limit: int = 300) -> str: return flat if len(flat) <= limit else "..." + flat[-limit:] -def git(repo: Path, *args: str) -> str: +def git_bytes(repo: Path, *args: str) -> bytes: + """A git command's stdout exactly as git wrote it (a ``-z`` listing keeps every name's leading and trailing + spaces).""" done = subprocess.run(["git", "-C", str(repo), *args], stdin=subprocess.DEVNULL, capture_output=True, timeout=120) if done.returncode != 0: raise RuntimeError(f"git {args[0]}: {tail(done.stderr.decode('utf-8', 'replace'), 200)}") - return done.stdout.decode("utf-8", "replace").strip() + return done.stdout + + +def git(repo: Path, *args: str) -> str: + return git_bytes(repo, *args).decode("utf-8", "replace").strip() # ------------------------------------------------------------------------------------------------------------- plan @@ -400,8 +406,11 @@ def build_contracts(plan: dict, repo: Path, thread_budget: float) -> tuple[dict, canary["evolution"]["evaluation"] = {"checks": {"canary": {"argv": [sys.executable, "-c", "pass"], "timeout_seconds": 60}}, "required": ["canary"], "max_age_seconds": 3600} - budget = canary["evolution"]["budget"] - budget.update({"max_candidates": 2, "max_evaluations": 2, "max_seconds": min(budget.get("max_seconds", 900), 900)}) + budget = canary["evolution"].get("budget") + if isinstance(budget, dict): # a whole contract without one is left for check_budget to refuse + max_seconds = budget.get("max_seconds", 900) + max_seconds = max_seconds if isinstance(max_seconds, (int, float)) and not isinstance(max_seconds, bool) else 900 + budget.update({"max_candidates": 2, "max_evaluations": 2, "max_seconds": min(max_seconds, 900)}) return main, canary @@ -548,19 +557,21 @@ def check_lane(env) -> Check: "third-party routing switch set", {"lane": "api", "cost_basis": "billed"}) -def scan_paths(repo: Path, extra: list) -> list[Path]: +def scan_paths(repo: Path, extra: list) -> tuple[list[Path], bool]: """Where the token must not be: every tracked and untracked (not ignored) file of the repository, the usual - credential files beside it and beside the host checkout whether ignored or not, and ``extra``.""" - paths = set() + credential files beside it and beside the host checkout whether ignored or not, and ``extra``; and whether git + listed the repository's files. The listing is split on NUL from git's own bytes, so a name that starts or ends + with a space is opened under its own name.""" + paths, listed = set(), True try: - names = git(repo, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0") - paths |= {repo / name for name in names if name} - except (RuntimeError, OSError): - pass + names = git_bytes(repo, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split(b"\0") + paths |= {repo / os.fsdecode(name) for name in names if name} + except (RuntimeError, OSError, subprocess.SubprocessError): + listed = False for root in {repo.resolve(), ROOT}: paths |= {root / name for name in SECRET_FILES} | set(root.glob(".env.*")) paths |= {Path(p) for p in extra} - return sorted(p for p in paths if p.is_file()) + return sorted(p for p in paths if p.is_file()), listed def holds(path: Path, needle: bytes, block: int = 1 << 20) -> bool: @@ -602,16 +613,23 @@ def check_token(env, repo: Path, plan_path: Path | None) -> Check: return Check("githits token", FAIL, f"{githits.TOKEN_ENV} is not set in the environment") if not token.isprintable() or any(ch.isspace() for ch in token): return Check("githits token", FAIL, f"{githits.TOKEN_ENV} holds whitespace or a control character") - found, read, skipped = files_holding(token, scan_paths(repo, [plan_path] if plan_path else [])) + paths, listed = scan_paths(repo, [plan_path] if plan_path else []) + found, read, skipped = files_holding(token, paths, MAX_SCAN_BYTES) if found: names = ", ".join(str(p) for p in found[:5]) + (f" and {len(found) - 5} more" if len(found) > 5 else "") return Check("githits token", FAIL, f"the token is written in {names}: remove it there and keep it in the " "environment only") - unread = f"; {skipped} not read (over {MAX_SCAN_BYTES >> 20} MB or unreadable)" if skipped else "" + facts = {"files_scanned": read, "files_unread": skipped, "repository_listed": listed} + if not listed: + return Check("githits token", FAIL, "git could not list the repository's files, so they were not scanned for " + "the token", facts) + if skipped: + return Check("githits token", FAIL, f"{skipped} file(s) could not be scanned for the token (over " + f"{MAX_SCAN_BYTES >> 20} MB or unreadable): move them out of the " + "repository or make them readable", facts) return Check("githits token", PASS, f"in the environment only: in none of the {read} files scanned (the " "repository's tracked and untracked files, .env*, .mcp.json, local Claude " - f"settings, the plan){unread}", - {"files_scanned": read, "files_unread": skipped}) + "settings, the plan)", facts) def check_doctor(sy: Syberlabs) -> Check: @@ -776,7 +794,8 @@ def unmetered(events: list) -> str | None: def spend_of(events: list, contracts: dict) -> float | None: """Host-observed dollars over the threads in ``events``: a thread whose calls crossed the meter at a model the prices name counts its dollars, and one where nothing crossed it counts nothing. A model the prices do not name, - or a call the meter missed (``unmetered``), makes the whole sum unknown.""" + a call the meter missed (``unmetered``), or a thread with a spend record the host did not observe (a call that + returned a result but no metered usage) makes the whole sum unknown.""" if unmetered(events): return None total = 0.0 @@ -785,7 +804,7 @@ def spend_of(events: list, contracts: dict) -> float | None: if not item.cost.known: return None total += float(item.cost.value) - elif item.cost_basis == "declared": + elif item.cost_basis == "declared" or item.cost.basis: # spend recorded, but not by the host's meter return None return round(total, 6) @@ -1502,6 +1521,7 @@ def finish(self) -> None: text = json.dumps(receipt, indent=2, sort_keys=True) for name, value in credentials: text = text.replace(value, f"[{name} redacted]") + self.receipt = receipt = json.loads(text) # the table prints what the file holds, never the credential (out / "receipt.json").write_text(text + "\n", encoding="utf-8") say("") say(render(receipt, out)) @@ -1570,18 +1590,21 @@ def main(argv: list | None = None, *, ask: Ask | None = None, runner: Runner | N exclude_from_git(repo, out if args.out else runs_dir) dry = DryRun(args, out, seed, ask or input, runner or subprocess_runner) try: - return dry.run() + code = dry.run() except KeyboardInterrupt: dry.receipt["verdict"] = "ABORTED" dry.receipt["reasons"].append("interrupted by the operator") - return 1 + code = 1 except Exception as exc: # a fault here or in a command: the receipt says so, the traceback follows on stderr traceback.print_exc() dry.receipt["verdict"] = FAIL dry.receipt["reasons"].append(f"internal error: {type(exc).__name__}: {tail(str(exc), 200)}") - return 1 + code = 1 finally: dry.finish() + if code == 0 and dry.receipt.get("verdict") != PASS: # finish's credential scan can still fail a passing run + code = 1 + return code if __name__ == "__main__": diff --git a/tests/test_pilot_dry_run.py b/tests/test_pilot_dry_run.py index 955cebba..aece068d 100644 --- a/tests/test_pilot_dry_run.py +++ b/tests/test_pilot_dry_run.py @@ -650,6 +650,64 @@ def test_the_operator_must_confirm_before_anything_is_spent(self): self.assertEqual(self.runs(self.bin), []) +class ReviewFindings(DryRunCase): + """The six findings of the 2026-11-07 review of #143, each pinned.""" + + def test_unmetered_spend_on_a_thread_is_unknown_not_zero(self): + item = lambda basis, hashes: mock.Mock(cost_basis=basis, cost=metrics.Metric(None, hashes, "")) + with mock.patch.object(pilot, "unmetered", return_value=None): + with mock.patch.object(metrics, "work_items", return_value=[item("none", ("h1",))]): + self.assertIsNone(pilot.spend_of([], {}), "a spend record the host did not observe is unknown") + with mock.patch.object(metrics, "work_items", return_value=[item("none", ())]): + self.assertEqual(pilot.spend_of([], {}), 0.0, "a thread nothing crossed the meter for counts nothing") + self.assertIn("unknown", pilot.gate(None, 1.0, 8.0)) + + def test_the_token_check_fails_when_any_file_was_not_scanned(self): + paths, _ = pilot.scan_paths(self.root, []) + limit = max(p.stat().st_size for p in paths if p.is_file()) + with self.subTest("a file over the scan limit"), mock.patch.object(pilot, "MAX_SCAN_BYTES", limit): + big = self.root / "big.bin" + big.write_bytes(b"0" * (limit + 1)) + check = pilot.check_token(os.environ, self.root, None) + self.assertEqual((check.status, check.data["files_unread"]), ("FAIL", 1), check.detail) + big.unlink() + with self.subTest("git could not list the repository"), \ + mock.patch.object(pilot, "git_bytes", side_effect=RuntimeError("git ls-files: broken")): + check = pilot.check_token(os.environ, self.root, None) + self.assertEqual((check.status, check.data["repository_listed"]), ("FAIL", False), check.detail) + with self.subTest("a name that starts with a space"): + odd = self.root / " aaa.txt" + odd.write_text(f"token={self.token}\n", encoding="utf-8") + check = pilot.check_token(os.environ, self.root, None) + self.assertEqual(check.status, "FAIL", check.detail) + self.assertIn(" aaa.txt", check.detail) + odd.unlink() + self.assertEqual(pilot.check_token(os.environ, self.root, None).status, "PASS") + + def test_a_whole_contract_without_a_budget_is_refused_by_the_budget_check_not_a_crash(self): + main, _ = pilot.build_contracts(self.plan_doc(), ROOT, 1.0) + del main["evolution"]["budget"] + own, canary = pilot.build_contracts(self.plan_doc(prices=None, contract=main), ROOT, 1.0) + self.assertNotIn("budget", canary["evolution"]) + self.assertEqual(pilot.check_budget(8.0, own, 6).status, "FAIL") + + def plan_doc(self, **changes): + return Pieces.plan(self, **changes) + + def test_a_credential_found_at_the_end_fails_the_exit_code_and_never_prints(self): + def run(dry): + (dry.out / "leak.txt").write_text(self.token, encoding="utf-8") + dry.receipt["verdict"] = "PASS" + dry.receipt["reasons"].append(f"a reason that quotes {self.token}") + return 0 + with mock.patch.object(pilot.DryRun, "run", run): + code, printed, receipt = self.dry_run() + self.assertEqual((code, receipt["verdict"]), (1, "FAIL"), "the exit code follows the final verdict") + self.assertNotIn(self.token, printed, "the table prints the redacted receipt") + self.assertNotIn(self.token, (self.out / "receipt.json").read_text(encoding="utf-8")) + self.assertIn("pilot dry run: FAIL", printed) + + class OneCommand(DryRunCase): def test_the_script_as_the_operator_runs_it(self): """``python scripts/pilot_dry_run.py --plan ...``: every ``syberlabs`` command a subprocess of this Python,