Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 38 additions & 18 deletions scripts/pilot_dry_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -249,11 +249,17 @@ def tail(text: str, limit: int = 300) -> str:
return flat if len(flat) <= limit else "..." + flat[-limit:]


def git(repo: Path, *args: str) -> str:
def git_bytes(repo: Path, *args: str) -> bytes:
"""A git command's stdout exactly as git wrote it (a ``-z`` listing keeps every name's leading and trailing
spaces)."""
done = subprocess.run(["git", "-C", str(repo), *args], stdin=subprocess.DEVNULL, capture_output=True, timeout=120)
if done.returncode != 0:
raise RuntimeError(f"git {args[0]}: {tail(done.stderr.decode('utf-8', 'replace'), 200)}")
return done.stdout.decode("utf-8", "replace").strip()
return done.stdout


def git(repo: Path, *args: str) -> str:
return git_bytes(repo, *args).decode("utf-8", "replace").strip()


# ------------------------------------------------------------------------------------------------------------- plan
Expand Down Expand Up @@ -405,8 +411,11 @@ def build_contracts(plan: dict, repo: Path, thread_budget: float) -> tuple[dict,
canary["evolution"]["evaluation"] = {"checks": {"canary": {"argv": [sys.executable, "-c", "pass"],
"timeout_seconds": 60}},
"required": ["canary"], "max_age_seconds": 3600}
budget = canary["evolution"]["budget"]
budget.update({"max_candidates": 2, "max_evaluations": 2, "max_seconds": min(budget.get("max_seconds", 900), 900)})
budget = canary["evolution"].get("budget")
if isinstance(budget, dict): # a whole contract without one is left for check_budget to refuse
max_seconds = budget.get("max_seconds", 900)
max_seconds = max_seconds if isinstance(max_seconds, (int, float)) and not isinstance(max_seconds, bool) else 900
budget.update({"max_candidates": 2, "max_evaluations": 2, "max_seconds": min(max_seconds, 900)})
return main, canary


Expand Down Expand Up @@ -561,19 +570,21 @@ def check_lane(env) -> Check:
{"lane": LANE, "cost_basis": "billed"})


def scan_paths(repo: Path, extra: list) -> list[Path]:
def scan_paths(repo: Path, extra: list) -> tuple[list[Path], bool]:
"""Where the token must not be: every tracked and untracked (not ignored) file of the repository, the usual
credential files beside it and beside the host checkout whether ignored or not, and ``extra``."""
paths = set()
credential files beside it and beside the host checkout whether ignored or not, and ``extra``; and whether git
listed the repository's files. The listing is split on NUL from git's own bytes, so a name that starts or ends
with a space is opened under its own name."""
paths, listed = set(), True
try:
names = git(repo, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split("\0")
paths |= {repo / name for name in names if name}
except (RuntimeError, OSError):
pass
names = git_bytes(repo, "ls-files", "-z", "--cached", "--others", "--exclude-standard").split(b"\0")
paths |= {repo / os.fsdecode(name) for name in names if name}
except (RuntimeError, OSError, subprocess.SubprocessError):
listed = False
for root in {repo.resolve(), ROOT}:
paths |= {root / name for name in SECRET_FILES} | set(root.glob(".env.*"))
paths |= {Path(p) for p in extra}
return sorted(p for p in paths if p.is_file())
return sorted(p for p in paths if p.is_file()), listed


def holds(path: Path, needle: bytes, block: int = 1 << 20) -> bool:
Expand Down Expand Up @@ -615,16 +626,23 @@ def check_token(env, repo: Path, plan_path: Path | None) -> Check:
return Check("githits token", FAIL, f"{githits.TOKEN_ENV} is not set in the environment")
if not token.isprintable() or any(ch.isspace() for ch in token):
return Check("githits token", FAIL, f"{githits.TOKEN_ENV} holds whitespace or a control character")
found, read, skipped = files_holding(token, scan_paths(repo, [plan_path] if plan_path else []))
paths, listed = scan_paths(repo, [plan_path] if plan_path else [])
found, read, skipped = files_holding(token, paths, MAX_SCAN_BYTES)
if found:
names = ", ".join(str(p) for p in found[:5]) + (f" and {len(found) - 5} more" if len(found) > 5 else "")
return Check("githits token", FAIL, f"the token is written in {names}: remove it there and keep it in the "
"environment only")
unread = f"; {skipped} not read (over {MAX_SCAN_BYTES >> 20} MB or unreadable)" if skipped else ""
facts = {"files_scanned": read, "files_unread": skipped, "repository_listed": listed}
if not listed:
return Check("githits token", FAIL, "git could not list the repository's files, so they were not scanned for "
"the token", facts)
if skipped:
return Check("githits token", FAIL, f"{skipped} file(s) could not be scanned for the token (over "
f"{MAX_SCAN_BYTES >> 20} MB or unreadable): move them out of the "
"repository or make them readable", facts)
return Check("githits token", PASS, f"in the environment only: in none of the {read} files scanned (the "

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Token preflight passes when the scan did not read the tree

check_token returns PASS when the token was absent from the files it managed to read. A tracked file over 20MB is skipped and the row stays PASS, so the run can still start. If git ls-files fails, the handler drops the listing, and the PASS line still says the tracked files were scanned.

How to see it: A tracked file one byte over MAX_SCAN_BYTES whose bytes contain the token. Status is PASS and files_unread is 1. A git ls-files failure with the token in a tracked file also returns PASS.

Smallest fix: Return FAIL when any candidate file was not read or the file list could not be listed.

"repository's tracked and untracked files, .env*, .mcp.json, local Claude "
f"settings, the plan){unread}",
{"files_scanned": read, "files_unread": skipped})
"settings, the plan)", facts)


def check_doctor(sy: Syberlabs) -> Check:
Expand Down Expand Up @@ -791,7 +809,8 @@ def unmetered(events: list) -> str | None:
def spend_of(events: list, contracts: dict) -> float | None:
"""Host-observed dollars over the threads in ``events``: a thread whose calls crossed the meter at a model the
prices name counts its dollars, and one where nothing crossed it counts nothing. A model the prices do not name,
or a call the meter missed (``unmetered``), makes the whole sum unknown."""
a call the meter missed (``unmetered``), or a thread with a spend record the host did not observe (a call that
returned a result but no metered usage) makes the whole sum unknown."""
if unmetered(events):
return None
total = 0.0
Expand All @@ -800,7 +819,7 @@ def spend_of(events: list, contracts: dict) -> float | None:
if not item.cost.known:
return None
total += float(item.cost.value)
elif item.cost_basis == "declared":
elif item.cost_basis == "declared" or item.cost.basis: # spend recorded, but not by the host's meter
return None
return round(total, 6)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Unpriced api calls count as zero against the cap

spend_of returns 0.0 for a finished thread whose cost the metrics fold marks unknown (cost_basis none). gate then starts the next arm. A search that ends with spend.host_observed false is outside UNMETERED, so a billed call that produced a result and estimated_usd but no assistant usage does not stop the schedule. The canary rejects that same record.

How to see it: A thread with search_finished.spend.host_observed false, dollars 0, signal.lane api, and signal.estimated_usd 3.5. cost.known is false, spend_of is 0.0, and gate returns None.

Smallest fix: If a thread's cost is not a known host-observed amount, return None from spend_of, as it already does for declared usage.


Expand Down Expand Up @@ -1516,6 +1535,7 @@ def finish(self) -> None:
text = json.dumps(receipt, indent=2, sort_keys=True)
for name, value in credentials:
text = text.replace(value, f"[{name} redacted]")
self.receipt = receipt = json.loads(text) # the table prints what the file holds, never the credential
(out / "receipt.json").write_text(text + "\n", encoding="utf-8")
say("")
say(render(json.loads(text), out))
Expand Down
58 changes: 58 additions & 0 deletions tests/test_pilot_dry_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,64 @@ def run(self):
self.assertNotIn(secret, text)


class ReviewFindings(DryRunCase):
"""The six findings of the 2026-11-07 review of #143, each pinned."""

def test_unmetered_spend_on_a_thread_is_unknown_not_zero(self):
item = lambda basis, hashes: mock.Mock(cost_basis=basis, cost=metrics.Metric(None, hashes, ""))
with mock.patch.object(pilot, "unmetered", return_value=None):
with mock.patch.object(metrics, "work_items", return_value=[item("none", ("h1",))]):
self.assertIsNone(pilot.spend_of([], {}), "a spend record the host did not observe is unknown")
with mock.patch.object(metrics, "work_items", return_value=[item("none", ())]):
self.assertEqual(pilot.spend_of([], {}), 0.0, "a thread nothing crossed the meter for counts nothing")
self.assertIn("unknown", pilot.gate(None, 1.0, 8.0))

def test_the_token_check_fails_when_any_file_was_not_scanned(self):
paths, _ = pilot.scan_paths(self.root, [])
limit = max(p.stat().st_size for p in paths if p.is_file())
with self.subTest("a file over the scan limit"), mock.patch.object(pilot, "MAX_SCAN_BYTES", limit):
big = self.root / "big.bin"
big.write_bytes(b"0" * (limit + 1))
check = pilot.check_token(os.environ, self.root, None)
self.assertEqual((check.status, check.data["files_unread"]), ("FAIL", 1), check.detail)
big.unlink()
with self.subTest("git could not list the repository"), \
mock.patch.object(pilot, "git_bytes", side_effect=RuntimeError("git ls-files: broken")):
check = pilot.check_token(os.environ, self.root, None)
self.assertEqual((check.status, check.data["repository_listed"]), ("FAIL", False), check.detail)
with self.subTest("a name that starts with a space"):
odd = self.root / " aaa.txt"
odd.write_text(f"token={self.token}\n", encoding="utf-8")
check = pilot.check_token(os.environ, self.root, None)
self.assertEqual(check.status, "FAIL", check.detail)
self.assertIn(" aaa.txt", check.detail)
odd.unlink()
self.assertEqual(pilot.check_token(os.environ, self.root, None).status, "PASS")

def test_a_whole_contract_without_a_budget_is_refused_by_the_budget_check_not_a_crash(self):
main, _ = pilot.build_contracts(self.plan_doc(), ROOT, 1.0)
del main["evolution"]["budget"]
own, canary = pilot.build_contracts(self.plan_doc(prices=None, contract=main), ROOT, 1.0)
self.assertNotIn("budget", canary["evolution"])
self.assertEqual(pilot.check_budget(8.0, own, 6).status, "FAIL")

def plan_doc(self, **changes):
return Pieces.plan(self, **changes)

def test_a_credential_found_at_the_end_fails_the_exit_code_and_never_prints(self):
def run(dry):
(dry.out / "leak.txt").write_text(self.token, encoding="utf-8")
dry.receipt["verdict"] = "PASS"
dry.receipt["reasons"].append(f"a reason that quotes {self.token}")
return 0
with mock.patch.object(pilot.DryRun, "run", run):
code, printed, receipt = self.dry_run()
self.assertEqual((code, receipt["verdict"]), (1, "FAIL"), "the exit code follows the final verdict")
self.assertNotIn(self.token, printed, "the table prints the redacted receipt")
self.assertNotIn(self.token, (self.out / "receipt.json").read_text(encoding="utf-8"))
self.assertIn("pilot dry run: FAIL", printed)


class OneCommand(DryRunCase):
def test_the_script_as_the_operator_runs_it(self):
"""``python scripts/pilot_dry_run.py --plan ...``: every ``syberlabs`` command a subprocess of this Python,
Expand Down
Loading