diff --git a/CMakeLists.txt b/CMakeLists.txt index 6239fb2ea..902e3441c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1294,6 +1294,9 @@ if(Python3_Interpreter_FOUND) # the same check where a lane actually runs it: with the bump staged and HEAD still behind, where a HEAD-only # read of the gitlink passed a registry recording the pin being replaced (engine#1738's post-commit failure) add_test(NAME registry_pins_staged COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/registry_pin_staged_test.py) + # ModelScope as a fallback source for pinned files, against a fake mirror: a same-named file with other bytes is + # refused, an unverified fetch never lands, a non-LFS pin is compared by its git blob id, and no request carries a token + add_test(NAME pin_mirror COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/pin_mirror_test.py) # the 1.9 measurement tool's own paths, on synthetic packs: the "trying next" line it shipped broken (its # only real pair verified on the first candidate, so the line never ran) and the residual tolerance it # accepts and refuses at. Fixtures are ~200 KB each and built from the rule, so they can be wrong on purpose. diff --git a/docs/model-pins.md b/docs/model-pins.md index be21b3a6f..c1f379091 100644 --- a/docs/model-pins.md +++ b/docs/model-pins.md @@ -145,6 +145,43 @@ makes the run's first line say so, and adding it is `--pin` plus a `used_for` th A repo of `.1bp` packs is not pinned here, and a private repo never is (next section) — both are filtered out, so the census cannot go permanently red over something this file must not contain. +## A fallback source: ModelScope + +ModelScope (modelscope.cn) mirrors many of the repos we pin, under the same id, and its public file listing +gives each file's sha256. So a mirror can stand in for Hugging Face exactly when its file carries the pinned oid, +whatever the file is called. `tools/pin_mirror.py` checks that and fetches through it. It is anonymous: +the public listing and download need no token, and the tool never sends one. The owner's ModelScope key stays +off this box, and nothing here uploads. + +```bash +tools/pin_mirror.py --check # every pin outside our own orgs: is the mirror the pinned file? +tools/pin_mirror.py --check --repo unsloth/Qwen3.8-27B-GGUF +tools/pin_mirror.py --fetch REPO FILE --dest DIR # download from the mirror; kept only if sha256 == the pin's oid +``` + +`--check` reports each pinned file as one of four states: + +- `match`: the mirror may serve it. +- `DIFFERS`: other bytes under the pinned name, so do not use it. +- `missing`. +- no mirror under that id. + +A non-LFS pin records a git blob id, which a sha256 cannot be compared with. For those, a small text file +(≤ 1 MiB) is downloaded and its blob id computed. `--fetch` hashes while it streams, resumes a `.part` +(the hash covers the bytes already on disk), and deletes a file that is not the pinned one rather than moving it +into place. Exit codes: + +- **0** everything compared matches, or the fetch verified. +- **1** a mirror differs, or the fetch did not verify. +- **2** usage, or no listing read. + +Measured 2026-11-16 against `models/pins.json` (72 pins outside our orgs): + +- 63 are mirrored under the same id, and all 76 of their pinned files match, including + `unsloth/Qwen3.8-27B-GGUF`'s UD3.1 `Qwen3.8-27B-UD-Q4_K_XL.gguf` (`2a5f3565…`). +- 9 have no mirror under that id: two RichardErkhov repos, three mradermacher repos, mys, nomic-ai, nvidia and pqnet. +- One real fetch, `unsloth/Qwen3-0.6B-GGUF` `Qwen3-0.6B-Q4_K_M.gguf` (397 MB, 36 s), verified against its pin. + ## Build inputs The model pins above are one kind of input. A build or CI run also takes inputs written inline in workflows and diff --git a/tests/pin_mirror_test.py b/tests/pin_mirror_test.py new file mode 100644 index 000000000..456600144 --- /dev/null +++ b/tests/pin_mirror_test.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tools/pin_mirror.py against a fake ModelScope (no network, ~0.1 s). + +The decisive cases: a mirror file under the pinned name with other bytes must be refused as a source, a fetched file +that is not the pinned one must never be moved into place, a non-LFS pin (a git blob id) is compared by its blob id +and not waved through on a sha256, and no request carries a token. +""" +import hashlib +import io +import json +import os +import sys +import tempfile +import urllib.error + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "tools")) +import pin_mirror as M # noqa: E402 + +fails = 0 + + +def check(ok, what): + global fails + print(("ok " if ok else "FAIL ") + what) + if not ok: + fails += 1 + + +def sha(b): + return hashlib.sha256(b).hexdigest() + + +GOOD, OTHER, TEXT = b"W" * 5000, b"V" * 5000, b'{"chat_template": "t"}\n' +SERVED = {} # (repo, path) -> bytes the fake mirror serves +LISTED = {} # repo -> listing rows +SEEN = [] # every request's headers + + +class Resp(io.BytesIO): + def __init__(self, b, status=200): + super().__init__(b) + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *a): + self.close() + + +def fake_urlopen(req, timeout=0): + # patched under the tool's own _open, so SEEN is what would really go on the wire + url, headers = req.full_url, {k.lower(): v for k, v in req.header_items()} + SEEN.append(headers) + if "/api/v1/models/" in url: + repo = url.split("/api/v1/models/")[1].split("/repo/files")[0] + if repo not in LISTED: + raise urllib.error.HTTPError(url, 404, "nf", {}, None) + return Resp(json.dumps({"Code": 200, "Data": {"Files": LISTED[repo]}}).encode()) + repo, path = url.split("/models/")[1].split("/resolve/master/") + b = SERVED[(repo, path)] + rng = headers.get("range") + if rng: + start = int(rng.split("=")[1].rstrip("-")) + return Resp(b[start:], 206) + return Resp(b) + + +M.urllib.request.urlopen = fake_urlopen + + +def files(repo, rows): + LISTED[repo] = [{"Path": p, "Size": len(b), "Sha256": sha(b), "IsLFS": lfs, "Type": "blob"} + for p, b, lfs in rows] + for p, b, _ in rows: + SERVED[(repo, p)] = b + + +with tempfile.TemporaryDirectory() as tmp: + pins = {"pins": { + "up/Good-GGUF": {"files": {"m.gguf": {"oid": sha(GOOD), "size": len(GOOD), "lfs": True}, + "tokenizer_config.json": {"oid": M.gitblob(TEXT), "size": len(TEXT), "lfs": False}}}, + "up/Moved-GGUF": {"files": {"m.gguf": {"oid": sha(GOOD), "size": len(GOOD), "lfs": True}}}, + "up/Partial-GGUF": {"files": {"a.gguf": {"oid": sha(GOOD), "size": len(GOOD), "lfs": True}, + "b.gguf": {"oid": sha(GOOD), "size": len(GOOD), "lfs": True}}}, + "up/Absent-GGUF": {"files": {"m.gguf": {"oid": sha(GOOD), "size": len(GOOD), "lfs": True}}}, + "1bit-MONSTER/Ours": {"files": {"m.gguf": {"oid": sha(GOOD), "size": len(GOOD), "lfs": True}}}, + }} + pf = os.path.join(tmp, "pins.json") + json.dump(pins, open(pf, "w")) + files("up/Good-GGUF", [("m.gguf", GOOD, True), ("tokenizer_config.json", TEXT, False)]) + files("up/Moved-GGUF", [("m.gguf", OTHER, True)]) + files("up/Partial-GGUF", [("a.gguf", GOOD, True)]) + out = os.path.join(tmp, "r.json") + + # 1. one mirror serves other bytes -> rc 1, and that file is named. + rc = M.main(["--pins", pf, "--check", "--json", out]) + r = json.load(open(out)) + st = {(x["repo"], x["path"]): x["state"] for x in r["rows"]} + check(rc == 1, f"a mirror with other bytes under the pinned name fails the check (rc {rc})") + check(st.get(("up/Moved-GGUF", "m.gguf")) == "DIFFERS", "the differing file is reported DIFFERS") + check(st.get(("up/Good-GGUF", "m.gguf")) == "match", "a matching LFS file is a match") + check(st.get(("up/Good-GGUF", "tokenizer_config.json")) == "match", + "a non-LFS pin is compared by its git blob id (downloaded), and matches") + check(st.get(("up/Partial-GGUF", "b.gguf")) == "missing", "a pinned file the mirror lacks is missing") + check(r["unmirrored"] == ["up/Absent-GGUF"], f"a repo with no mirror is listed, not failed ({r['unmirrored']})") + check(not any(x["repo"].startswith("1bit-MONSTER/") for x in r["rows"]) and "1bit-MONSTER/Ours" not in r["unmirrored"], + "our own orgs are not asked") + + # 2. the non-LFS check really is the blob id: a sha256 of the text equal to nothing in the pin must not pass. + SERVED[("up/Good-GGUF", "tokenizer_config.json")] = TEXT + b" " + rc = M.main(["--pins", pf, "--check", "--repo", "up/Good-GGUF", "--json", out]) + st = {x["path"]: x["state"] for x in json.load(open(out))["rows"]} + check(rc == 1 and st["tokenizer_config.json"] == "DIFFERS", f"a changed non-LFS file is DIFFERS (rc {rc})") + SERVED[("up/Good-GGUF", "tokenizer_config.json")] = TEXT + + # 3. --repo limited to clean repos -> rc 0. + rc = M.main(["--pins", pf, "--check", "--repo", "up/Good-GGUF", "up/Absent-GGUF"]) + check(rc == 0, f"every compared file matches -> rc 0 (rc {rc})") + + # 4. fetch: the pinned bytes land in place; other bytes are deleted, never moved into place. + dest = os.path.join(tmp, "dl") + rc = M.main(["--pins", pf, "--fetch", "up/Good-GGUF", "m.gguf", "--dest", dest]) + check(rc == 0 and open(os.path.join(dest, "m.gguf"), "rb").read() == GOOD, f"a verified fetch lands (rc {rc})") + rc = M.main(["--pins", pf, "--fetch", "up/Moved-GGUF", "m.gguf", "--dest", os.path.join(tmp, "dl2")]) + left = os.listdir(os.path.join(tmp, "dl2")) + check(rc == 1 and left == [], f"a fetch of other bytes is refused and leaves nothing (rc {rc}, left {left})") + + # 4b. a non-LFS pin is fetched and verified by its git blob id (streamed), and refused when the bytes differ. + rc = M.main(["--pins", pf, "--fetch", "up/Good-GGUF", "tokenizer_config.json", "--dest", os.path.join(tmp, "dlt")]) + check(rc == 0 and open(os.path.join(tmp, "dlt", "tokenizer_config.json"), "rb").read() == TEXT, + f"a non-LFS fetch verifies by blob id (rc {rc})") + SERVED[("up/Good-GGUF", "tokenizer_config.json")] = TEXT + b" " + rc = M.main(["--pins", pf, "--fetch", "up/Good-GGUF", "tokenizer_config.json", "--dest", os.path.join(tmp, "dlt2")]) + check(rc == 1 and os.listdir(os.path.join(tmp, "dlt2")) == [], f"a changed non-LFS fetch is refused (rc {rc})") + SERVED[("up/Good-GGUF", "tokenizer_config.json")] = TEXT + + # 5. a partial download resumes and the hash covers the bytes already on disk. + d3 = os.path.join(tmp, "dl3") + os.makedirs(d3) + open(os.path.join(d3, "m.gguf.part"), "wb").write(GOOD[:1234]) + rc = M.main(["--pins", pf, "--fetch", "up/Good-GGUF", "m.gguf", "--dest", d3]) + check(rc == 0 and open(os.path.join(d3, "m.gguf"), "rb").read() == GOOD, f"a resumed fetch verifies (rc {rc})") + check(any(h.get("range") == "bytes=1234-" for h in SEEN), "the resume asked for the remaining range only") + open(os.path.join(d3, "m.gguf.part"), "wb").write(OTHER[:1234]) # a corrupt partial must not pass as resumed + os.remove(os.path.join(d3, "m.gguf")) + rc = M.main(["--pins", pf, "--fetch", "up/Good-GGUF", "m.gguf", "--dest", d3]) + check(rc == 1 and not os.path.exists(os.path.join(d3, "m.gguf")), f"a corrupt partial is caught by the hash (rc {rc})") + + # 6. unknown inputs are rc 2; no request ever carries a token. + check(M.main(["--pins", pf, "--fetch", "up/Good-GGUF", "nope.gguf", "--dest", dest]) == 2, "an unpinned file is rc 2") + check(M.main(["--pins", pf, "--check", "--repo", "no/Such"]) == 2, "an unpinned repo is rc 2") + check(SEEN and not any("authorization" in h for h in SEEN) and all(h.get("user-agent") == M.UA for h in SEEN), + f"no request carries a token ({len(SEEN)} requests)") + +print("pin_mirror_test: " + ("PASS" if fails == 0 else f"{fails} FAILED")) +sys.exit(1 if fails else 0) diff --git a/tools/pin_mirror.py b/tools/pin_mirror.py new file mode 100644 index 000000000..524d04ba3 --- /dev/null +++ b/tools/pin_mirror.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ModelScope as a fallback download source for pinned models, checked against the pin (docs/model-pins.md). + +usage: tools/pin_mirror.py --check [--repo REPO...] [--pins FILE] [--out FILE] [--json FILE] + tools/pin_mirror.py --fetch REPO FILE --dest DIR [--pins FILE] + +A pin names Hugging Face bytes by their sha256 (`lfs.oid`). ModelScope (modelscope.cn) mirrors many of the same +repos under the same id, and its public file listing reports each file's sha256, so a mirror can stand in for +Hugging Face exactly when its file carries the pinned oid -- and must not otherwise, whatever its name says. +Everything here is anonymous: the public listing and download need no token, and this tool never sends one. + + --check for every pin outside our own orgs (or --repo ...): list the same id on ModelScope and compare each + pinned file. States: match (the mirror may serve it), DIFFERS (it serves other bytes: do not use it), + missing (the mirror has no file at that path), no mirror (no repo under that id). A non-LFS pin + records a git blob id, which a sha256 cannot be compared with, so a small text file is downloaded + (<= 1 MiB) and its blob id computed. + --fetch download one pinned file from the mirror into DIR, hashing while it streams; the file is moved into + place only if its hash is the pin's oid (sha256 for an LFS file, the git blob id for the rest), and + deleted otherwise. A partial download resumes. + +Exit codes: 0 every compared file matches (--check) or the file was fetched and verified (--fetch); 1 a mirror +serves different bytes for a pinned file (--check), or the fetched file is not the pinned one (--fetch); 2 usage, +an unknown repo or file, or no mirror listing could be read at all. +""" +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PINS = os.path.join(ROOT, "models", "pins.json") +ENDPOINT = os.environ.get("MODELSCOPE_ENDPOINT", "https://modelscope.cn") +OWN = ("1bit-MONSTER", "bong-water-water-bong") # we publish these; a mirror of them is not a source +UA = "1bit-engine-pin-mirror" +SMALL = 1 << 20 +CHUNK = 16 << 20 + + +def _open(url: str, headers: dict | None = None, timeout: int = 120): + # no Authorization header, ever: the fallback must work for anyone, and no token leaves this box + return urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA, **(headers or {})}), + timeout=timeout) + + +def listing(repo: str) -> dict[str, dict] | None: + """{path: {"size", "sha256", "lfs"}} of the mirror's master, or None when ModelScope has no such repo.""" + q = urllib.parse.quote(repo, safe="/") + url = f"{ENDPOINT}/api/v1/models/{q}/repo/files?Revision=master&Recursive=true" + last = None + for attempt in range(4): + try: + with _open(url) as r: + d = json.load(r) + break + except urllib.error.HTTPError as e: + if e.code == 404: + return None + last = e + except Exception as e: # transient: back off, then give up + last = e + time.sleep(1.5 * (attempt + 1)) + else: + raise last + if d.get("Code") != 200: + return None + return {f["Path"]: {"size": f.get("Size"), "sha256": f.get("Sha256") or "", "lfs": bool(f.get("IsLFS"))} + for f in (d.get("Data") or {}).get("Files") or [] if f.get("Type") != "tree"} + + +def resolve_url(repo: str, path: str) -> str: + return f"{ENDPOINT}/models/{urllib.parse.quote(repo, safe='/')}/resolve/master/{urllib.parse.quote(path)}" + + +def gitblob(b: bytes) -> str: + return hashlib.sha1(b"blob %d\0" % len(b) + b).hexdigest() + + +def compare(repo: str, pin: dict, files: dict[str, dict]) -> list[dict]: + rows = [] + for path, want in sorted(pin["files"].items()): + f = files.get(path) + row = {"repo": repo, "path": path, "want": want["oid"], "size": want.get("size")} + if f is None: + row["state"] = "missing" + elif want.get("lfs", True): + row["have"] = f["sha256"] + row["state"] = ("no sha256 listed" if not f["sha256"] else + "match" if f["sha256"] == want["oid"] and f["size"] in (None, want.get("size")) else + "DIFFERS") + elif (f["size"] or 0) > SMALL: + row["state"] = "not compared (non-LFS file over 1 MiB)" + else: + with _open(resolve_url(repo, path)) as r: + blob = r.read(SMALL + 1) + row["have"] = gitblob(blob) + row["state"] = "match" if row["have"] == want["oid"] else "DIFFERS" + rows.append(row) + return rows + + +def check(pins: dict, repos: list[str]) -> tuple[list[dict], list[str], list[str]]: + rows, unmirrored, unread = [], [], [] + for repo in repos: + try: + files = listing(repo) + except Exception as e: + unread.append(f"{repo}: {e}") + continue + if files is None: + unmirrored.append(repo) + continue + try: + rows += compare(repo, pins[repo], files) + except Exception as e: + unread.append(f"{repo}: {e}") + return rows, unmirrored, unread + + +def report(rows, unmirrored, unread, total) -> str: + day = datetime.datetime.now(datetime.timezone.utc).date().isoformat() + by = {} + for r in rows: + by.setdefault(r["state"], []).append(r) + mirrored = total - len(unmirrored) - len(unread) + repos_ok = mirrored - len({r["repo"] for r in rows if r["state"] != "match"}) + lines = [f"# ModelScope mirrors of our pins, {day}", "", + f"{total} repos asked: {mirrored} mirrored ({repos_ok} with every pinned " + f"file matching), {len(unmirrored)} not on ModelScope under the same id, {len(unread)} not read.", + "Files: " + ", ".join(f"{k} {len(v)}" for k, v in sorted(by.items())), ""] + bad = [r for r in rows if r["state"] == "DIFFERS"] + if bad: + lines += ["## Mirrors serving other bytes than the pin (not a fallback for these files)", ""] + lines += [f"- {r['repo']} `{r['path']}`: pin {r['want'][:12]}…, mirror {r.get('have', '')[:12]}…" for r in bad] + lines.append("") + other = [r for r in rows if r["state"] not in ("match", "DIFFERS")] + if other: + lines += ["## Not usable as a fallback", ""] + lines += [f"- {r['repo']} `{r['path']}`: {r['state']}" for r in other] + lines.append("") + if unmirrored: + lines += ["## No ModelScope repo under the same id", ""] + [f"- {r}" for r in unmirrored] + [""] + if unread: + lines += ["## Could not be read", ""] + [f"- {u}" for u in unread] + [""] + return "\n".join(lines) + + +def fetch(pins: dict, repo: str, path: str, dest: str) -> int: + want = pins[repo]["files"][path] + os.makedirs(dest, exist_ok=True) + final = os.path.join(dest, os.path.basename(path)) + part = final + ".part" + have = os.path.getsize(part) if os.path.exists(part) else 0 + h = hashlib.sha256() if want.get("lfs", True) else None + if have and h is not None: + with open(part, "rb") as f: # resume: the hash has to cover the bytes already on disk + for b in iter(lambda: f.read(CHUNK), b""): + h.update(b) + hdr = {"Range": f"bytes={have}-"} if have else {} + with _open(resolve_url(repo, path), hdr, timeout=300) as r: + if have and r.status != 206: # the server ignored the range: start over rather than append a second copy + have, h = 0, (hashlib.sha256() if h is not None else None) + with open(part, "ab" if have else "wb") as out: + for b in iter(lambda: r.read(CHUNK), b""): + out.write(b) + if h is not None: + h.update(b) + size = os.path.getsize(part) + if h is not None: + got = h.hexdigest() + else: # a git blob id needs the size first, so it is a second streamed pass, never the whole file in memory + g = hashlib.sha1(b"blob %d\0" % size) + with open(part, "rb") as f: + for b in iter(lambda: f.read(CHUNK), b""): + g.update(b) + got = g.hexdigest() + if got != want["oid"] or (want.get("size") is not None and size != want["size"]): + os.remove(part) + print(f"pin_mirror: {repo} {path} from ModelScope is {got[:12]}… {size} B, the pin is " + f"{want['oid'][:12]}… {want.get('size')} B: deleted, not used", file=sys.stderr) + return 1 + os.replace(part, final) + print(f"{final}: {size} B, {got} = pin {repo} {path}") + return 0 + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--pins", default=PINS) + ap.add_argument("--check", action="store_true") + ap.add_argument("--repo", nargs="+", metavar="REPO") + ap.add_argument("--fetch", nargs=2, metavar=("REPO", "FILE")) + ap.add_argument("--dest") + ap.add_argument("--out") + ap.add_argument("--json") + a = ap.parse_args(argv) + try: + pins = json.load(open(a.pins, encoding="utf-8"))["pins"] + except (OSError, ValueError, KeyError) as e: + print(f"pin_mirror: cannot read {a.pins}: {e}", file=sys.stderr) + return 2 + if a.fetch: + repo, path = a.fetch + if not a.dest: + ap.error("--fetch needs --dest") + if repo not in pins or path not in pins[repo]["files"]: + print(f"pin_mirror: {repo} {path} is not pinned", file=sys.stderr) + return 2 + return fetch(pins, repo, path, a.dest) + if not a.check: + ap.error("give --check or --fetch") + repos = a.repo or sorted(r for r in pins if r.split("/")[0] not in OWN) + unknown = [r for r in repos if r not in pins] + if unknown: + print(f"pin_mirror: not pinned: {', '.join(unknown)}", file=sys.stderr) + return 2 + rows, unmirrored, unread = check(pins, repos) + text = report(rows, unmirrored, unread, len(repos)) + print(text) + if a.out: + open(a.out, "w", encoding="utf-8").write(text + "\n") + if a.json: + json.dump({"rows": rows, "unmirrored": unmirrored, "unread": unread}, open(a.json, "w"), indent=1) + if unread and len(unread) == len(repos): + return 2 + return 1 if any(r["state"] == "DIFFERS" for r in rows) else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))