diff --git a/CMakeLists.txt b/CMakeLists.txt index a9ed75674..d41380057 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1291,6 +1291,9 @@ if(Python3_Interpreter_FOUND) # engine#1745 review: pin() reads the INDEX (a staged bump) and refuses an UNMERGED index instead of returning the # merge base. Hermetic: a scratch git repo and a copy of the tool, git only, ~0.1 s. add_test(NAME registry_pin_index COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/registry_pin_index_test.py) + # the box half of the pin watch (UNSLOTH-UD31): a box copy or a re-host that is not the pinned file, on synthetic + # pins and files; the cases the daily pin watch stayed green through (same size other bytes, a stale cached hash) + add_test(NAME pin_holdings COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/pin_holdings_test.py) # the same check where a lane actually runs it: with the bump staged and HEAD still behind, where a HEAD-only # read of the gitlink passed a registry recording the pin being replaced (engine#1738's post-commit failure) add_test(NAME registry_pins_staged COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/registry_pin_staged_test.py) diff --git a/docs/model-pins.md b/docs/model-pins.md index be21b3a6f..d6c2e1a7d 100644 --- a/docs/model-pins.md +++ b/docs/model-pins.md @@ -135,6 +135,46 @@ ships or was converted from it, and that reason is what makes the entry reviewab per **file**, never per model — `Qwen3-4B`'s GGUF in our org and the quant the box serves carry two different templates, both legitimate. +## What the box holds + +A pin records the bytes we **mean** to serve; the daily check compares it with Hugging Face, which cannot see +the box. So a pin can move to a new upstream file while the box keeps serving the old one, and every run stays +green. That happened: `unsloth/Qwen3.8-27B-GGUF` was pinned at the UD3.1 `Qwen3.8-27B-UD-Q4_K_XL.gguf` on +2026-11-12 (`2a5f3565…`, 17,659,407,264 B) while `~/models/qwen38/UD-Q4_K_XL/` and our re-host +`1bit-MONSTER/Qwen3.8-27B-UD-Q4_K_XL-MTP` still held the earlier file (`3f227079…`, 17,559,178,144 B). + +`tools/pin_holdings.py` closes both halves: + +* **holdings** — [`models/holdings.json`](../models/holdings.json) maps a pinned file to its copy on the box. + Each copy's sha256 (the git blob id for a non-LFS file, which is what the pin records there) is compared with + the pin's oid. The hashes are cached by path, size, mtime and inode (`~/.cache/1bit/pin-holdings-sha256.json`), + so a daily run re-reads only a file that changed, and a size mismatch is reported without reading the file. + A path in the map is a promise that the box serves the pinned bytes: add one when a pinned file lands on the + box, and change it in the same PR that moves a default path. +* **re-hosts** — a pin whose `used_for` says `upstream of ` is compared, file by file of the same name, + with ``'s own pin. Both oids are in `models/pins.json`, so this half needs neither the box nor the + network (`--rehost-only`, which CI can run), and the box run reports it too. The fix is a re-upload, and + publishing is the owner's. + +Its exit codes are its own, not `pin_watch.py`'s (whose table is above): + +- **0** every holding and every re-host matches its pin; +- **1** a holding differs from its pin or is missing, or a re-host holds a different file than its upstream pin; +- **2** it could not run: usage, an unreadable `pins.json`/`holdings.json`, no engine checkout where the unit + looks, or a holding naming a pin or file `models/pins.json` does not have (a wrong map is never clean). + +On the box it runs daily from `scripts/1bit-pin-holdings.{service,timer}` (05:00 UTC) under `~/lb/thermal-run.sh`, +from the engine checkout named by `ONEBIT_ENGINE` in the unit (default `%h/wt/engine-main`; set it at install to a +checkout of this repository that follows `main`, which `~/1bit-MONSTER` on strixhalo is not). A missing checkout +exits 2 with a line saying so, so "cannot run" never reads as "a pin differs". Exit 1 marks the unit failed and the +report is `~/.1bit/logs/pin-holdings-latest.md`. + +```bash +tools/pin_holdings.py # both halves; report on stdout +tools/pin_holdings.py --rehost-only # pins.json only (CI) +tools/pin_holdings.py --out r.md --json r.json +``` + ## The repos we publish (the other direction) `--check` iterates over pins, so a repo that was never pinned is invisible to it — including one we publish diff --git a/models/holdings.json b/models/holdings.json new file mode 100644 index 000000000..3d5988ba1 --- /dev/null +++ b/models/holdings.json @@ -0,0 +1,15 @@ +{ + "about": "The pinned files the development box (strixhalo) keeps, and where. tools/pin_holdings.py compares each copy's sha256 with its pin's oid (docs/model-pins.md, 'What the box holds'). A path here is a promise that the box serves the pinned bytes: add one when a pinned file lands on the box, and change it in the same PR that moves a default path.", + "host": "strixhalo", + "holdings": [ + {"pin": "unsloth/Qwen3.8-27B-GGUF", "file": "Qwen3.8-27B-UD-Q4_K_XL.gguf", "path": "~/models/qwen38/UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf"}, + {"pin": "1bit-MONSTER/Qwen3.8-27B-DFlash2-GGUF", "file": "Qwen3.8-27B-DFlash2-q4k.gguf", "path": "~/models/qwen38/DFlash2/Qwen3.8-27B-DFlash2-q4k.gguf"}, + {"pin": "1bit-MONSTER/Qwen3.8-27B-DFlash2-GGUF", "file": "Qwen3.8-27B-DFlash2-q8_0.gguf", "path": "~/models/qwen38/DFlash2/Qwen3.8-27B-DFlash2-q8_0.gguf"}, + {"pin": "unsloth/flint-GGUF", "file": "flint-UD-Q4_K_XL.gguf", "path": "~/models/flint/flint-UD-Q4_K_XL.gguf"}, + {"pin": "unsloth/Qwen3-0.6B-GGUF", "file": "Qwen3-0.6B-Q4_K_M.gguf", "path": "~/models/Qwen3-0.6B-Q4_K_M.gguf"}, + {"pin": "unsloth/Qwen3-4B-GGUF", "file": "Qwen3-4B-Q4_K_M.gguf", "path": "~/models/Qwen3-4B-Q4_K_M.gguf"}, + {"pin": "1bit-MONSTER/ZAYA1-8B-GGUF", "file": "zaya1-8b-Q4_K_M.gguf", "path": "~/models/zaya1-8b-Q4_K_M.gguf"}, + {"pin": "Qwen/Qwen2.5-0.5B-Instruct-GGUF", "file": "qwen2.5-0.5b-instruct-q4_k_m.gguf", "path": "~/models/gpnpu/Qwen2.5-0.5B/qwen2.5-0.5b-instruct-q4_k_m.gguf"}, + {"pin": "1bit-MONSTER/ZR1-1.5B-GGUF", "file": "ZR1-1.5B-Q4_K_M.gguf", "path": "~/models/zyphra-orig/quant/ZR1-1.5B/ZR1-1.5B-Q4_K_M.gguf"} + ] +} diff --git a/scripts/1bit-pin-holdings.service b/scripts/1bit-pin-holdings.service new file mode 100644 index 000000000..4a3aefbcf --- /dev/null +++ b/scripts/1bit-pin-holdings.service @@ -0,0 +1,42 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# 1bit-pin-holdings.service — do the model files on this box match models/pins.json? (docs/model-pins.md) +# +# One-shot. Exit 1 (a box copy or a re-host is not the pinned file) marks the unit failed, which is the +# alert; the report is ~/.1bit/logs/pin-holdings-latest.md either way. Hashes are cached by +# path/size/mtime/inode, so a day where nothing changed reads no model file. +# +# install -m644 scripts/1bit-pin-holdings.{service,timer} ~/.config/systemd/user/ # then set ONEBIT_ENGINE (below) +# systemctl --user daemon-reload +# systemctl --user enable --now 1bit-pin-holdings.timer +# +# ONEBIT_ENGINE must name a checkout of the ENGINE repository (1bit-MONSTER/engine) that follows main; set it at +# install. On strixhalo ~/1bit-MONSTER is a different repository (no tools/pin_holdings.py), so it is not the default. +# A missing checkout exits 2 with a line saying so: "cannot run" must not look like exit 1, "a pin differs". +# The run goes through ~/lb/thermal-run.sh (the box rule for CPU-heavy jobs): the first run hashes ~70 GB on one core. + +[Unit] +Description=1bit pin holdings: the box's model files match their pins + +[Service] +Type=oneshot +Environment=ONEBIT_ENGINE=%h/wt/engine-main +Nice=19 +IOSchedulingClass=idle +ExecStartPre=/usr/bin/mkdir -p %h/.1bit/logs +ExecStart=/bin/sh -c 'e="$ONEBIT_ENGINE"; for f in tools/pin_holdings.py models/pins.json models/holdings.json; do [ -f "$e/$f" ] || { echo "pin-holdings: $e/$f is missing: ONEBIT_ENGINE must be an engine checkout that follows main"; exit 2; }; done; cd "$e" && exec %h/lb/thermal-run.sh /usr/bin/python3 -I tools/pin_holdings.py --out %h/.1bit/logs/pin-holdings-latest.md' +StandardOutput=journal +StandardError=journal diff --git a/scripts/1bit-pin-holdings.timer b/scripts/1bit-pin-holdings.timer new file mode 100644 index 000000000..3b69eb60f --- /dev/null +++ b/scripts/1bit-pin-holdings.timer @@ -0,0 +1,27 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# 1bit-pin-holdings.timer — 05:00 UTC daily (after the 04:30 census watch), persisted across downtime. + +[Unit] +Description=Daily 1bit pin holdings check + +[Timer] +OnCalendar=*-*-* 05:00:00 UTC +Persistent=true +Unit=1bit-pin-holdings.service + +[Install] +WantedBy=timers.target diff --git a/tests/pin_holdings_test.py b/tests/pin_holdings_test.py new file mode 100644 index 000000000..ceacb39d2 --- /dev/null +++ b/tests/pin_holdings_test.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tools/pin_holdings.py on synthetic pins, holdings and box files (hermetic, ~0.5 s). + +The decisive cases are the ones the pin watch was green through: a box copy with the pinned size but other +bytes (only a hash sees it), a cached hash that must not survive a rewrite of the file, and a re-host whose pin +records a different file than its upstream's (Qwen3.8-27B-UD-Q4_K_XL, 2026-11-12..16). +""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.path.join(HERE, "..", "tools", "pin_holdings.py") + +fails = 0 + + +def check(ok, what): + global fails + print(("ok " if ok else "FAIL ") + what) + if not ok: + fails += 1 + + +def sha256(b): + return hashlib.sha256(b).hexdigest() + + +def gitblob(b): + return hashlib.sha1(b"blob %d\0" % len(b) + b).hexdigest() + + +def run(tmp, *extra, holdings="holdings.json"): + r = subprocess.run([sys.executable, TOOL, "--pins", os.path.join(tmp, "pins.json"), + "--holdings", os.path.join(tmp, holdings), "--cache", os.path.join(tmp, "cache.json"), + "--json", os.path.join(tmp, "out.json"), *extra], capture_output=True, text=True) + out = json.load(open(os.path.join(tmp, "out.json"))) if os.path.exists(os.path.join(tmp, "out.json")) else {} + if os.path.exists(os.path.join(tmp, "out.json")): + os.remove(os.path.join(tmp, "out.json")) + return r, out + + +def state(out, path): + return next((h["state"] for h in out.get("holdings") or [] if h["path"] == path), None) + + +with tempfile.TemporaryDirectory() as tmp: + good, text = b"G" * 4096, b'{"chat_template": "x"}\n' + old27, new27 = b"o" * 1000, b"n" * 1100 + pins = {"pins": { + "up/Model-GGUF": {"commit": "a", "pinned": "d", "used_for": ["upstream of us/Model-Rehost"], + "files": {"m.gguf": {"oid": sha256(new27), "size": len(new27), "lfs": True}, + "sub/head.gguf": {"oid": sha256(good), "size": len(good), "lfs": True}, + "renamed.gguf": {"oid": sha256(b"r"), "size": 1, "lfs": True}}}, + "us/Model-Rehost": {"commit": "b", "pinned": "d", "used_for": ["published: re-host"], + "files": {"m.gguf": {"oid": sha256(old27), "size": len(old27), "lfs": True}, + "head.gguf": {"oid": sha256(good), "size": len(good), "lfs": True}, + "other-name.gguf": {"oid": sha256(b"z"), "size": 1, "lfs": True}}}, + "x/Text": {"commit": "c", "pinned": "d", "used_for": [], + "files": {"tokenizer_config.json": {"oid": gitblob(text), "size": len(text), "lfs": False}}}, + }} + json.dump(pins, open(os.path.join(tmp, "pins.json"), "w")) + box = os.path.join(tmp, "box") + os.makedirs(box) + paths = {} + for name, data in {"head.gguf": good, "tok.json": text, "m-old.gguf": old27, "m-new.gguf": new27, + "same-size.gguf": b"X" * len(good)}.items(): + paths[name] = os.path.join(box, name) + open(paths[name], "wb").write(data) + + def holdings(rows): + json.dump({"holdings": rows}, open(os.path.join(tmp, "holdings.json"), "w")) + + base = [{"pin": "up/Model-GGUF", "file": "sub/head.gguf", "path": paths["head.gguf"]}, + {"pin": "x/Text", "file": "tokenizer_config.json", "path": paths["tok.json"]}, + {"pin": "up/Model-GGUF", "file": "m.gguf", "path": paths["m-new.gguf"]}] + + # 1. every holding matches (LFS sha256 and a non-LFS git blob id); the re-host drift still fails the run. + holdings(base) + r, out = run(tmp) + check([h["state"] for h in out["holdings"]] == ["ok", "ok", "ok"], f"matching copies are ok ({out.get('holdings')})") + check(r.returncode == 1 and len(out["rehosts"]) == 1, f"the re-host behind its upstream fails the run (rc {r.returncode})") + rh = out["rehosts"][0] if out.get("rehosts") else {} + check(rh.get("file") == "m.gguf" and rh.get("rehost_oid") == sha256(old27), + "the drifted re-host row names the file and the oid the re-host holds") + check(not any(x.get("file") in ("sub/head.gguf", "renamed.gguf") for x in out.get("rehosts", [])), + "a same-named file with the same oid (in another folder) is not drift; a renamed one is not compared") + check("needs a re-upload" in r.stdout, "the report says the re-host needs a re-upload") + + # 2. the re-host fixed: everything clean, rc 0. + pins["pins"]["us/Model-Rehost"]["files"]["m.gguf"] = dict(pins["pins"]["up/Model-GGUF"]["files"]["m.gguf"]) + json.dump(pins, open(os.path.join(tmp, "pins.json"), "w")) + r, out = run(tmp) + check(r.returncode == 0 and out["rehosts"] == [], f"all copies and re-hosts match -> rc 0 (rc {r.returncode})") + check("0 hashed" in r.stdout, "the second run takes every hash from the cache") + + # 3. the box holds the OLD file under the pinned name: a size mismatch, reported without reading it. + holdings(base[:2] + [{"pin": "up/Model-GGUF", "file": "m.gguf", "path": paths["m-old.gguf"]}]) + r, out = run(tmp) + check(r.returncode == 1 and state(out, paths["m-old.gguf"]) == "size differs", + f"the old file at the pinned path is drift (rc {r.returncode}, {state(out, paths['m-old.gguf'])})") + check("have" not in out["holdings"][2], "a size mismatch is decided without hashing the file") + + # 4. same size, other bytes: only the hash can see it. + holdings([{"pin": "up/Model-GGUF", "file": "sub/head.gguf", "path": paths["same-size.gguf"]}]) + r, out = run(tmp) + check(r.returncode == 1 and state(out, paths["same-size.gguf"]) == "oid differs", + f"same size, different bytes -> oid differs (rc {r.returncode})") + + # 5. a cached hash does not outlive a rewrite of the file (same size, new mtime). + holdings(base[:1]) + r, out = run(tmp) + check(r.returncode == 0, "the head copy is ok before the rewrite") + time.sleep(0.01) + open(paths["head.gguf"], "wb").write(b"H" * len(good)) + r, out = run(tmp) + check(r.returncode == 1 and state(out, paths["head.gguf"]) == "oid differs", + f"rewritten in place at the same size -> re-hashed and caught (rc {r.returncode})") + + # 6. a missing copy is drift, not a skip. + holdings([{"pin": "x/Text", "file": "tokenizer_config.json", "path": os.path.join(box, "gone.json")}]) + r, out = run(tmp) + check(r.returncode == 1 and out["holdings"][0]["state"] == "missing", f"a missing copy fails (rc {r.returncode})") + + # 7. a map entry pins.json cannot resolve is rc 2, never clean. + for bad in ({"pin": "no/Such", "file": "m.gguf", "path": paths["m-new.gguf"]}, + {"pin": "up/Model-GGUF", "file": "nope.gguf", "path": paths["m-new.gguf"]}): + holdings([bad]) + r, out = run(tmp) + check(r.returncode == 2 and out.get("errors"), f"unresolvable holding {bad['pin']}:{bad['file']} -> rc 2 " + f"(rc {r.returncode})") + + # 8. --rehost-only needs no holdings file and reads no box file. + r, out = run(tmp, "--rehost-only", holdings="absent.json") + check(r.returncode == 0 and out["holdings"] is None, f"--rehost-only runs without a holdings map (rc {r.returncode})") + pins["pins"]["up/Model-GGUF"]["used_for"].append("upstream of us/Not-Pinned") + json.dump(pins, open(os.path.join(tmp, "pins.json"), "w")) + r, out = run(tmp, "--rehost-only", holdings="absent.json") + check(r.returncode == 1 and any(x["state"] == "re-host not pinned" for x in out["rehosts"]), + "an 'upstream of' naming an unpinned re-host is reported") + +print("pin_holdings_test: " + ("PASS" if fails == 0 else f"{fails} FAILED")) +sys.exit(1 if fails else 0) diff --git a/tools/pin_holdings.py b/tools/pin_holdings.py new file mode 100644 index 000000000..87be11913 --- /dev/null +++ b/tools/pin_holdings.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Do the files we hold match the pins? The box half of the pin watch (docs/model-pins.md). + +usage: tools/pin_holdings.py [--pins FILE] [--holdings FILE] [--cache FILE] [--out FILE] [--json FILE] + [--rehost-only] + +tools/pin_watch.py --check compares each pin with what Hugging Face serves. It cannot see the box, so a pin +can be bumped to a new upstream file while the box keeps serving the old one, and every daily run stays +green. That happened: unsloth/Qwen3.8-27B-GGUF was pinned at the UD3.1 Qwen3.8-27B-UD-Q4_K_XL.gguf on +2026-11-12 (oid 2a5f3565..., 17,659,407,264 B) while ~/models/qwen38/UD-Q4_K_XL/ and our re-host +1bit-MONSTER/Qwen3.8-27B-UD-Q4_K_XL-MTP still held the earlier file (3f227079..., 17,559,178,144 B). + +Two checks, one report: + + holdings each entry of models/holdings.json names a pin, a file of that pin, and the path of our copy on + the box. The copy's sha256 (the git blob id for a non-LFS file, which is what the pin records + for those) is compared with the pin's oid. Hashes are cached by path, size, mtime and inode + (--cache), so a daily run re-reads only a file that changed; a size mismatch is reported + without reading the file at all. + re-hosts a pin whose used_for says "upstream of " is compared, file by file of the same name, + with 's own pin. Both oids come from models/pins.json, so this half needs neither the + box nor the network, and --rehost-only runs it alone (CI can). + +Exit codes: 0 every holding and every re-host matches, 1 a drift (a holding differs from its pin or is +missing, or a re-host holds a different file than its upstream pin), 2 usage, or a holding names a pin or a +file pins.json does not have (the map is wrong, which must not read as clean). +""" +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PINS = os.path.join(ROOT, "models", "pins.json") +HOLDINGS = os.path.join(ROOT, "models", "holdings.json") +CACHE = os.path.expanduser("~/.cache/1bit/pin-holdings-sha256.json") +CHUNK = 16 << 20 + + +def load(path: str) -> dict: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def file_id(path: str, lfs: bool) -> str: + """sha256 of the bytes (an LFS oid), or the git blob id (sha1 of "blob \\0" + bytes).""" + h = hashlib.sha256() if lfs else hashlib.sha1(b"blob %d\0" % os.path.getsize(path)) + with open(path, "rb") as f: + while True: + b = f.read(CHUNK) + if not b: + break + h.update(b) + return h.hexdigest() + + +class Cache: + """Hash cache keyed by real path; an entry counts only while size, mtime and inode are what was hashed.""" + + def __init__(self, path: str | None): + self.path = path + self.rows = {} + if path and os.path.exists(path): + try: + self.rows = load(path) + except (OSError, ValueError): + self.rows = {} # an unreadable cache costs a re-hash, never a wrong answer + self.hashed = 0 + + def get(self, path: str, lfs: bool) -> str: + real = os.path.realpath(path) + st = os.stat(real) + key = f"{real}|{'sha256' if lfs else 'gitblob'}" + row = self.rows.get(key) + stamp = [st.st_size, st.st_mtime_ns, st.st_ino] + if row and row.get("stat") == stamp: + return row["id"] + got = file_id(real, lfs) + self.hashed += 1 + self.rows[key] = {"stat": stamp, "id": got} + return got + + def save(self) -> None: + if not self.path: + return + os.makedirs(os.path.dirname(self.path), exist_ok=True) + tmp = self.path + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + json.dump(self.rows, f, indent=1, sort_keys=True) + os.replace(tmp, self.path) + + +def check_holdings(pins: dict, holdings: list[dict], cache: Cache) -> tuple[list[dict], list[str]]: + """(rows, errors): one row per holding with its state; errors are map entries pins.json cannot resolve.""" + rows, errors = [], [] + for h in holdings: + repo, name = h.get("pin"), h.get("file") + path = os.path.expanduser(h.get("path") or "") + pin = pins.get(repo) + if pin is None: + errors.append(f"holding {path}: pin {repo!r} is not in pins.json") + continue + want = pin["files"].get(name) + if want is None: + errors.append(f"holding {path}: pin {repo} has no file {name!r}") + continue + row = {"pin": repo, "file": name, "path": path, "want": want["oid"], "want_size": want.get("size")} + if not os.path.exists(path): + row["state"] = "missing" + else: + size = os.path.getsize(os.path.realpath(path)) + row["size"] = size + if want.get("size") is not None and size != want["size"]: + row["state"] = "size differs" # no read needed: different bytes for certain + else: + row["have"] = cache.get(path, bool(want.get("lfs", True))) + row["state"] = "ok" if row["have"] == want["oid"] else "oid differs" + rows.append(row) + return rows, errors + + +def check_rehosts(pins: dict) -> list[dict]: + """Every same-named file where a re-host's pin and its upstream pin record different bytes.""" + out = [] + for up, pin in sorted(pins.items()): + for use in pin.get("used_for", []): + if not use.startswith("upstream of "): + continue + rehost = use[len("upstream of "):].strip() + mine = pins.get(rehost) + if mine is None: + out.append({"upstream": up, "rehost": rehost, "file": None, "state": "re-host not pinned"}) + continue + theirs = {os.path.basename(k): (k, v) for k, v in mine["files"].items()} + for path, want in sorted(pin["files"].items()): + hit = theirs.get(os.path.basename(path)) + if hit is None: + continue # a renamed or different quant in the re-host is not the same file to compare + if hit[1]["oid"] != want["oid"]: + out.append({"upstream": up, "rehost": rehost, "file": path, "rehost_file": hit[0], + "state": "re-host holds a different file", + "upstream_oid": want["oid"], "upstream_size": want.get("size"), + "rehost_oid": hit[1]["oid"], "rehost_size": hit[1].get("size")}) + return out + + +def gb(n) -> str: + return "?" if n is None else f"{n / 1e9:.2f} GB" + + +def report(hrows: list[dict] | None, rrows: list[dict], errors: list[str], hashed: int) -> str: + day = datetime.datetime.now(datetime.timezone.utc).date().isoformat() # UTC, as the CI run dates its report + bad = [r for r in hrows or [] if r["state"] != "ok"] + lines = [f"# Pin holdings, {day}", ""] + if hrows is None: + lines.append("Holdings: not checked (--rehost-only).") + else: + lines.append(f"Holdings: {len(hrows)} checked, {len(bad)} differ from their pin " + f"({hashed} hashed this run, the rest from the cache).") + lines.append(f"Re-hosts: {len(rrows)} differ from their upstream pin.") + if errors: + lines.append(f"Map errors: {len(errors)} (exit 2).") + lines.append("") + if bad: + lines += ["## Box copies that are not the pinned file", ""] + for r in bad: + have = r.get("have", "")[:12] or "-" + lines.append(f"- `{r['path']}` ({r['state']}): pin {r['pin']} `{r['file']}` is {r['want'][:12]}… " + f"{gb(r['want_size'])}; the box holds {have} {gb(r.get('size'))}") + lines.append("") + if rrows: + lines += ["## Re-hosts behind their upstream pin", ""] + for r in rrows: + if r["file"] is None: + lines.append(f"- {r['rehost']} (upstream {r['upstream']}): the re-host has no pin") + continue + lines.append(f"- {r['rehost']} `{r['rehost_file']}` is {r['rehost_oid'][:12]}… {gb(r['rehost_size'])}; " + f"upstream {r['upstream']} `{r['file']}` is pinned at {r['upstream_oid'][:12]}… " + f"{gb(r['upstream_size'])} — the re-host needs a re-upload (owner-only)") + lines.append("") + if errors: + lines += ["## Holdings map errors", ""] + [f"- {e}" for e in errors] + [""] + return "\n".join(lines) + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--pins", default=PINS) + ap.add_argument("--holdings", default=HOLDINGS) + ap.add_argument("--cache", default=CACHE, help="hash cache ('' disables it)") + ap.add_argument("--out", help="also write the Markdown report here") + ap.add_argument("--json", help="also write the rows as JSON here") + ap.add_argument("--rehost-only", action="store_true", help="only the pins.json half (no box files read)") + a = ap.parse_args(argv) + try: + pins = load(a.pins)["pins"] + except (OSError, ValueError, KeyError) as e: + print(f"pin_holdings: cannot read {a.pins}: {e}", file=sys.stderr) + return 2 + cache = Cache(a.cache or None) + hrows, errors = None, [] + if not a.rehost_only: + try: + holdings = load(a.holdings)["holdings"] + except (OSError, ValueError, KeyError) as e: + print(f"pin_holdings: cannot read {a.holdings}: {e}", file=sys.stderr) + return 2 + hrows, errors = check_holdings(pins, holdings, cache) + cache.save() + rrows = check_rehosts(pins) + text = report(hrows, rrows, errors, cache.hashed) + print(text) + if a.out: + with open(a.out, "w", encoding="utf-8") as f: + f.write(text + "\n") + if a.json: + with open(a.json, "w", encoding="utf-8") as f: + json.dump({"holdings": hrows, "rehosts": rrows, "errors": errors}, f, indent=1) + if errors: + return 2 + if rrows or any(r["state"] != "ok" for r in hrows or []): + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))