From a8a78a1f1ae2f8f9c9bf84cc91613a2bdc9b8469 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 18:19:35 -0400 Subject: [PATCH 1/4] pin-watch: the bump step's shell parses, and an unread upstream fails it instead of passing as a clean day The step that opens one PR per moved repo was committed with an unterminated quote (`git commit -m "pin: $repo at $newsha""`), so `bash -n` refuses the whole step. Nothing executed it until a pin moved, which is exactly what the first scheduled run would have done: moved=1, then a syntax error, no PR, and no bump for any repo. $ bash -n # before line 23: unexpected EOF while looking for matching `"' Two more defects in the same step, both of the "clean and could-not-check look alike" kind: a bump that failed for any reason was a warning and a `continue`, so a Hub outage produced a green run with no PR; and `git switch -c` aborts the run if the branch survived an earlier failure. `--bump` now returns 4 for the two cases that are a human decision (repo gone upstream, tracked file gone) and 1 for anything else, the step skips 4 with a warning and fails on anything else, and the branch is created with -C. --- .github/workflows/pin-watch.yml | 19 ++++++++++++++----- tools/pin_watch.py | 16 +++++++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pin-watch.yml b/.github/workflows/pin-watch.yml index 6a5d9a3de..afccbe516 100644 --- a/.github/workflows/pin-watch.yml +++ b/.github/workflows/pin-watch.yml @@ -82,17 +82,26 @@ jobs: if git ls-remote --exit-code origin "refs/heads/$branch" > /dev/null 2>&1; then echo "$branch already exists"; continue fi - git switch -c "$branch" main --quiet - if ! python3 tools/pin_watch.py --bump "$repo"; then - # a tracked file vanished upstream: the report says so, and a human decides - echo "::warning::$repo could not be bumped automatically (see the report)" + # -C, not -c: a bump that failed earlier in this run left its branch behind, and a + # "branch already exists" abort here would leave every later repo unbumped + git switch -C "$branch" main --quiet + rc=0 + python3 tools/pin_watch.py --bump "$repo" || rc=$? + if [ "$rc" -eq 4 ]; then + # gone upstream, or a tracked file did: the report says so, and a human decides + echo "::warning::$repo needs a human decision (see the report); no bump opened" + git checkout -- models/pins.json 2>/dev/null || true git switch main --quiet continue + elif [ "$rc" -ne 0 ]; then + # anything else is an unread upstream, which is not a clean day + echo "::error::$repo could not be bumped (rc=$rc): this run is not a clean one" + exit 1 fi python3 tools/pin_watch.py --pr-body "$repo" > body.md || true newsha=$(python3 -c 'import json,sys; print(json.load(open("models/pins.json"))["pins"][sys.argv[1]]["commit"][:8])' "$repo") git add models/pins.json - git commit -q -m "pin: $repo at $newsha"" + git commit -q -m "pin: $repo at $newsha" git push -q origin "$branch" gh pr create --base main --head "$branch" --title "Pin: $repo" --body-file body.md git switch main --quiet diff --git a/tools/pin_watch.py b/tools/pin_watch.py index 1de19c7fb..28b3677bd 100644 --- a/tools/pin_watch.py +++ b/tools/pin_watch.py @@ -37,7 +37,11 @@ --out FILE where --check writes its report (default pin-report.md). --self-test run against fixtures, no network. -Exit codes: 0 nothing moved, 1 a pin moved (or --bump refused), 2 usage, 3 --self-test failed. +Exit codes: 0 nothing moved, 1 a pin moved (or --check found nothing readable), 2 usage, +3 --self-test failed, 4 --bump needs a human (the repo or a tracked file is gone upstream). + +The workflow reads 4 as "skip this repo and say so", and any other non-zero as an unread upstream: +a bump that did not happen because the Hub was down is not a clean day either. """ from __future__ import annotations @@ -541,7 +545,8 @@ def fake_tree_missing(repo, rev="main"): return [v for k, v in trees["new"].items() if k != "tpl.gguf"] head, tree, commits_since = fake_head_new, fake_tree_missing, fake_commits - expect("--bump refuses a vanished file", main(["--bump", "fake/moved", "--pins", path]), 1) + expect("--bump refuses a vanished file (a human decides, not an error)", + main(["--bump", "fake/moved", "--pins", path]), 4) expect("--bump left the pin alone", load(path)["pins"]["fake/moved"]["commit"], "new") tree = fake_tree @@ -561,7 +566,8 @@ def fake_head_404(repo): gone_md = report(gone_changes, gone_notes, 1, False) expect("the report says gone upstream", "Gone upstream" in gone_md, True) expect("no all-clear over a 404", "Every pinned upstream repo is at" in gone_md, False) - expect("--bump refuses a gone repo", main(["--bump", "fake/gone", "--pins", path]), 1) + expect("--bump refuses a gone repo (a human decides, not an error)", + main(["--bump", "fake/gone", "--pins", path]), 4) expect("the gone pin is untouched", load(path)["pins"]["fake/gone"]["commit"], "new") # a run that could read nothing is exit 2 (the workflow fails on anything outside 0/1) and its @@ -616,13 +622,13 @@ def main(argv: list[str]) -> int: if e.code == 404: print(f"pin_watch: {a.bump}: gone upstream (404) — a delete, a rename or a privatisation; " f"decide, then edit or drop the pin", file=sys.stderr) - return 1 + return 4 raise missing = [p for p in pin["files"] if p not in at_new] if missing: print(f"pin_watch: {a.bump}: {len(missing)} tracked file(s) are gone upstream: " + ", ".join(missing) + " — decide, then edit the pin", file=sys.stderr) - return 1 + return 4 tpls = {p: remote_template(a.bump, now, p) for p in pin["files"]} doc["pins"][a.bump] = entry(a.bump, sorted(pin["files"]), pin["used_for"], templates=tpls) save(a.pins, doc) From 6d24a2526029b31a934825bc4bde35de5e61e789 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 18:19:35 -0400 Subject: [PATCH 2/4] tests/workflow_shell: every workflow's shell must parse, and the bump step must open one PR per moved repo The guard for the class, not just the instance: every `run:` block in .github/workflows/*.yml is lifted out of the text and run through `bash -n` (no PyYAML in CI), then the pin bump step is driven against a local bare origin with a stubbed `gh` and `python3`: one branch, one commit and one PR per moved repo, only models/pins.json in the commit, a repo that went upstream warned about and skipped, and an unread upstream failing the run. Against the pre-fix workflow it reports 9 failures; against this tree, 30 ok. --- CMakeLists.txt | 4 + tests/workflow_shell_test.py | 206 +++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 tests/workflow_shell_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt index c2ee15796..1eff6feed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -551,6 +551,10 @@ if(Python3_Interpreter_FOUND) # pin while going red every day unnoticed. The step is driven against a stubbed gh, with and # without an open issue. add_test(NAME bump_workflow_report COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/bump_workflow_report_test.py) + # every workflow's `run:` must parse, and the pin bump step must open one PR per moved repo while a + # repo that went upstream stays a human decision: the step shipped with an unterminated quote that + # nothing executed until a pin moved, i.e. until its first scheduled run (engine #1148) + add_test(NAME workflow_shell COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/workflow_shell_test.py) # a Q4NX directory's chunk layout (engine #587): synthetic containers, so the check a # published FastFlowLM model goes through is proved to fire without the model at hand. diff --git a/tests/workflow_shell_test.py b/tests/workflow_shell_test.py new file mode 100644 index 000000000..ae652188a --- /dev/null +++ b/tests/workflow_shell_test.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A workflow's shell must parse, and the pin bump step must open one PR per moved repo. + +pin-watch.yml's "Open one pin bump per changed repo" step was committed with an unterminated quote +(`git commit -m "pin: $repo at $newsha""`). Nothing executed that step until a pin actually moved, so a +syntax error that kills the whole step - and with it every bump PR - sat on main looking green: the +scheduled run at 05:43 UTC would have been its first reader. + +Two guarantees, both offline (CI installs no PyYAML, so the blocks are lifted out of the text): + + * every `run:` in every workflow parses under `bash -n` (the class, not just this instance); + * the pin bump step, driven against a local bare origin, a stubbed `gh` and a stubbed `python3`, opens + exactly one PR per moved repo, leaves a repo that went upstream to a human, and fails the run when an + upstream could not be read at all - an unread upstream is not a clean day. +""" +import json +import os +import re +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +WORKFLOWS = os.path.join(ROOT, ".github", "workflows") +STEP = "Open one pin bump per changed repo" +failures = [] + + +def check(what, cond, detail=""): + if cond: + print(f"ok {what}") + else: + print(f"FAIL {what}" + (f": {detail}" if detail else "")) + failures.append(what) + + +def blocks(text): + """[(line number, shell)] for every `run: |` block and every inline `run:` scalar.""" + lines = text.splitlines() + out, i = [], 0 + while i < len(lines): + m = re.match(r"^(\s*)(?:- )?run: (\||>[-+]?|)\s*$", lines[i]) + inline = re.match(r"^(\s*)(?:- )?run: (?!\||>)(\S.*)$", lines[i]) + if m: + indent, body, j = len(m.group(1)), [], i + 1 + while j < len(lines) and (not lines[j].strip() or len(lines[j]) - len(lines[j].lstrip()) > indent): + body.append(lines[j]) + j += 1 + while body and not body[-1].strip(): + body.pop() + cut = min((len(b) - len(b.lstrip()) for b in body if b.strip()), default=0) + out.append((i + 1, "\n".join(b[cut:] for b in body) + "\n")) + i = j + elif inline: + out.append((i + 1, inline.group(2))) + i += 1 + else: + i += 1 + return out + + +def step_script(text, name): + """The de-indented shell of the named step, or None.""" + lines = text.splitlines() + at = next((i for i, l in enumerate(lines) if f"- name: {name}" in l), None) + if at is None: + return None + found = blocks("\n".join(lines[at:])) + return found[0][1] if found else None + + +def sandbox(tmp, report): + """A bare origin plus a working clone carrying a pins file and the report the check step writes.""" + origin, work = os.path.join(tmp, "origin.git"), os.path.join(tmp, "work") + subprocess.run(["git", "init", "-q", "--bare", origin], check=True) + subprocess.run(["git", "clone", "-q", origin, work], check=True, capture_output=True) + for cmd in (["config", "user.email", "t@example.com"], ["config", "user.name", "t"]): + subprocess.run(["git", "-C", work] + cmd, check=True) + os.makedirs(os.path.join(work, "models")) + os.makedirs(os.path.join(work, "tools")) + pins = {"version": 1, "pins": {}} + for repo in ("ok/one", "ok/two", "human/gone", "broken/x"): + pins["pins"][repo] = {"commit": "a" * 40, "pinned": "2026-11-02", "used_for": ["t"], "files": {}} + with open(os.path.join(work, "models", "pins.json"), "w") as fh: + json.dump(pins, fh, indent=1) + open(os.path.join(work, "tools", "pin_watch.py"), "w").close() + with open(os.path.join(work, "pin-report.md"), "w") as fh: + fh.write("# Model pins: 4 of 4 moved (2026-11-05)\n\n") + for c in report: + fh.write(f"## `{c['repo']}`\n\n{c['body']}\n") + subprocess.run(["git", "-C", work, "add", "-A"], check=True) + subprocess.run(["git", "-C", work, "commit", "-qm", "init"], check=True) + subprocess.run(["git", "-C", work, "branch", "-M", "main"], check=True) + subprocess.run(["git", "-C", work, "push", "-q", "-u", "origin", "main"], check=True) + + stub = os.path.join(tmp, "stub") + os.makedirs(stub) + log = os.path.join(tmp, "gh.log") + with open(os.path.join(stub, "gh"), "w") as fh: + fh.write(f'#!/bin/sh\nprintf "%s\\n" "$*" >> "{log}"\necho https://example/pr/1\n') + # `python3` stands in for the tool's two network calls; the step's own `python3 -c` still runs for real + with open(os.path.join(stub, "python3"), "w") as fh: + fh.write(f'''#!/bin/sh +REAL={sys.executable} +if [ "$1" = tools/pin_watch.py ] && [ "$2" = --bump ]; then + case "$3" in + human/gone) echo "pin_watch: $3: gone upstream (404)" >&2; exit 4 ;; + broken/x) echo "pin_watch: $3: could not read the Hub" >&2; exit 1 ;; + esac + "$REAL" -c 'import json,sys; p="models/pins.json"; d=json.load(open(p)); d["pins"][sys.argv[1]]["commit"]="b"*40; json.dump(d,open(p,"w"),indent=1)' "$3" || exit 1 + exit 0 +fi +if [ "$1" = tools/pin_watch.py ] && [ "$2" = --pr-body ]; then + echo "body for $3"; exit 0 +fi +exec "$REAL" "$@" +''') + for f in ("gh", "python3"): + os.chmod(os.path.join(stub, f), 0o700) + return work, stub, log + + +def drive(script, work, stub, log): + env = dict(os.environ, PATH=stub + os.pathsep + os.environ.get("PATH", ""), GH_TOKEN="test") + p = subprocess.run(["bash", "-c", script], cwd=work, env=env, capture_output=True, text=True) + calls = open(log).read().splitlines() if os.path.exists(log) else [] + heads = subprocess.run(["git", "-C", work, "ls-remote", "--heads", "origin"], + capture_output=True, text=True).stdout.split() + return p, [c for c in calls if c.strip()], heads + + +# 1. every workflow's shell parses - the class the pin-watch step was caught by +for name in sorted(os.listdir(WORKFLOWS)): + if not name.endswith(".yml"): + continue + text = open(os.path.join(WORKFLOWS, name)).read() + found = blocks(text) + bad = [] + for line, shell in found: + p = subprocess.run(["bash", "-n"], input=shell, text=True, capture_output=True) + if p.returncode != 0: + bad.append(f"line {line}: {p.stderr.strip().splitlines()[-1]}") + check(f"{name}: {len(found)} run block(s) parse", not bad, "; ".join(bad)[:200]) + +# 2. the pin bump step: one PR per moved repo, a gone repo left to a human, an unread one fatal +text = open(os.path.join(WORKFLOWS, "pin-watch.yml")).read() +script = step_script(text, STEP) +check("pin-watch: has the pin bump step", script is not None) +if script: + p = subprocess.run(["bash", "-n"], input=script, text=True, capture_output=True) + check("pin-watch: the pin bump step's shell parses", p.returncode == 0, p.stderr.strip()[:200]) + + report = [{"repo": "ok/one", "body": "`a` -> `b`"}, + {"repo": "human/gone", "body": "**Gone upstream** — the Hub answers `404`."}, + {"repo": "ok/two", "body": "`a` -> `b`"}] + with tempfile.TemporaryDirectory() as tmp: + work, stub, log = sandbox(tmp, report) + p, calls, heads = drive(script, work, stub, log) + creates = [c for c in calls if c.startswith("pr create")] + check("pin-watch: a gone repo does not fail the run", p.returncode == 0, p.stderr.strip()[-200:]) + check("pin-watch: one 'gh pr create' per moved repo", len(creates) == 2, str(calls)) + check("pin-watch: the gone repo is warned about, not bumped", + "human/gone needs a human decision" in p.stdout + p.stderr) + branches = sorted(h.split("refs/heads/")[-1] for h in heads if "refs/heads/pin-watch/" in h) + check("pin-watch: one branch per moved repo, none for the gone one", len(branches) == 2, str(branches)) + for repo in ("ok/one", "ok/two"): + slug = repo.replace("/", "-").replace(".", "-") + branch = next((b for b in branches if b.startswith(f"pin-watch/{slug}-")), None) + check(f"pin-watch: {repo} has a pushed branch at main", branch is not None, str(branches)) + if branch is None: + continue + check(f"pin-watch: {repo}'s PR names that branch", + any(c.startswith("pr create") and f"--head {branch}" in c for c in creates), str(creates)) + msg = subprocess.run(["git", "-C", work, "log", "-1", "--format=%s", branch], + capture_output=True, text=True).stdout.strip() + check(f"pin-watch: {repo}'s commit says what it pins", msg == f"pin: {repo} at bbbbbbbb", msg) + files = subprocess.run(["git", "-C", work, "show", "--name-only", "--format=", branch], + capture_output=True, text=True).stdout.split() + check(f"pin-watch: {repo}'s commit touches only the pins file", files == ["models/pins.json"], str(files)) + + report = [{"repo": "broken/x", "body": "`a` -> `b`"}] + with tempfile.TemporaryDirectory() as tmp: + work, stub, log = sandbox(tmp, report) + p, calls, heads = drive(script, work, stub, log) + check("pin-watch: an unread upstream fails the run instead of a green no-PR day", + p.returncode != 0 and "could not be bumped" in p.stdout + p.stderr, f"rc={p.returncode}") + check("pin-watch: and opens nothing", not [c for c in calls if c.startswith("pr create")], str(calls)) + +if failures: + print(f"\n{len(failures)} check(s) failed") + sys.exit(1) +print("\nall checks passed") From 5771c1234fa8bf279d05181c2b6541e5dde1736e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 18:33:03 -0400 Subject: [PATCH 3/4] pin-watch: one sweep, N PR bodies (--check --json, --pr-body --from) The docstring said --pr-body prints the body "from the report --check just wrote", and it did not: it re-ran the whole sweep. At 89 s per sweep that is one full read of all 147 repos per bump PR, so a day when a vendor re-pushes many repos gets slower the more it has to report - and hammers the Hub exactly when the run must not fail. --check --json FILE writes that same sweep (changes + notes) beside the report, and --pr-body REPO --from FILE reads it instead of re-checking. Without --from it still recomputes, so an ad hoc single call reads the same as before. The workflow passes pin-report.json to both, so a moved-repo day costs one sweep. Measured on the committed pins: --check 89 s, then --pr-body --from 0.12 s, and the body is diff-identical to the 89 s recomputation. --self-test 38/38, including a --pr-body --from call with head()/tree() replaced by a stub that raises, so a hidden re-read would fail the test. The workflow rehearsal in tests/workflow_shell_test.py now fails the step if --from or its JSON is missing. --- .github/workflows/pin-watch.yml | 6 +++-- .gitignore | 4 +++ docs/model-pins.md | 4 ++- tests/workflow_shell_test.py | 4 +++ tools/pin_watch.py | 48 +++++++++++++++++++++++++++++++-- 5 files changed, 61 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pin-watch.yml b/.github/workflows/pin-watch.yml index afccbe516..3eb880538 100644 --- a/.github/workflows/pin-watch.yml +++ b/.github/workflows/pin-watch.yml @@ -61,7 +61,8 @@ jobs: set -uo pipefail # exit 1 means "a pin moved", which is the normal case this workflow exists for rc=0 - python3 tools/pin_watch.py --check --out pin-report.md || rc=$? + # the JSON beside the report is the same sweep, so the PR step below does not repeat it + python3 tools/pin_watch.py --check --out pin-report.md --json pin-report.json || rc=$? if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then exit "$rc"; fi echo "moved=$(grep -c '^## `' pin-report.md || true)" >> "$GITHUB_OUTPUT" cat pin-report.md >> "$GITHUB_STEP_SUMMARY" @@ -98,7 +99,8 @@ jobs: echo "::error::$repo could not be bumped (rc=$rc): this run is not a clean one" exit 1 fi - python3 tools/pin_watch.py --pr-body "$repo" > body.md || true + # no `|| true`: a body that could not be produced is a broken report, not an empty PR + python3 tools/pin_watch.py --pr-body "$repo" --from pin-report.json > body.md newsha=$(python3 -c 'import json,sys; print(json.load(open("models/pins.json"))["pins"][sys.argv[1]]["commit"][:8])' "$repo") git add models/pins.json git commit -q -m "pin: $repo at $newsha" diff --git a/.gitignore b/.gitignore index f63603d5d..08fc0ad18 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,7 @@ Testing/ # the HRX toolchain the build fetches (scripts/fetch-rocm-sdk.sh, config/hrx-toolchain.config): # a multi-gigabyte venv of TheRock's ROCm SDK at the pinned version, not part of the source third_party/toolchain/ + +# the model-pin sweep writes these where it runs (tools/pin_watch.py --check --json) +pin-report.md +pin-report.json diff --git a/docs/model-pins.md b/docs/model-pins.md index abc9cea29..ce5d8a734 100644 --- a/docs/model-pins.md +++ b/docs/model-pins.md @@ -105,7 +105,9 @@ a decision (re-take it under its new name, or drop the pin), not a diff. ```bash tools/pin_watch.py --check # the daily comparison; writes pin-report.md, exit 0/1/2 (above) tools/pin_watch.py --check --no-template # same, without the template range reads -tools/pin_watch.py --pr-body REPO # the bump PR body for one moved repo +tools/pin_watch.py --check --json sweep.json # ...and the same sweep as JSON +tools/pin_watch.py --pr-body REPO --from sweep.json # that repo's PR body, from that sweep: no second read +tools/pin_watch.py --pr-body REPO # without --from it repeats the sweep (one full read per repo) tools/pin_watch.py --pin REPO [REPO...] # print an entry for a repo at its current main (adding a pin) tools/pin_watch.py --bump REPO # rewrite one entry in models/pins.json (a reviewed change) tools/pin_watch.py --self-test # fixtures, no network diff --git a/tests/workflow_shell_test.py b/tests/workflow_shell_test.py index ae652188a..0b1241a95 100644 --- a/tests/workflow_shell_test.py +++ b/tests/workflow_shell_test.py @@ -102,6 +102,8 @@ def sandbox(tmp, report): fh.write("# Model pins: 4 of 4 moved (2026-11-05)\n\n") for c in report: fh.write(f"## `{c['repo']}`\n\n{c['body']}\n") + with open(os.path.join(work, "pin-report.json"), "w") as fh: + json.dump({"date": "2026-11-05", "changes": report}, fh) subprocess.run(["git", "-C", work, "add", "-A"], check=True) subprocess.run(["git", "-C", work, "commit", "-qm", "init"], check=True) subprocess.run(["git", "-C", work, "branch", "-M", "main"], check=True) @@ -125,6 +127,8 @@ def sandbox(tmp, report): exit 0 fi if [ "$1" = tools/pin_watch.py ] && [ "$2" = --pr-body ]; then + # the JSON beside the report is the sweep this must read, so a missing --from fails the step here + if [ "$4" != --from ] || [ ! -s "$5" ]; then echo "pr-body without the sweep: $*" >&2; exit 7; fi echo "body for $3"; exit 0 fi exec "$REAL" "$@" diff --git a/tools/pin_watch.py b/tools/pin_watch.py index 28b3677bd..b461c1ccc 100644 --- a/tools/pin_watch.py +++ b/tools/pin_watch.py @@ -30,11 +30,15 @@ every changed `.gguf` (the metadata header only, over an HTTP range request) and says whether the template changed, because a new revision may not be adopted before that is known. --no-template skips that read (the report then says it was not checked). - --pr-body R print the bump PR's body for repo R from the report --check just wrote. + --pr-body R print the bump PR's body for repo R and exit 0. With --from, it comes from the sweep + a --check wrote (`--json`); without it the sweep is repeated, one full Hub read per + repo. A repo that did not move is a usage error (3), not a body. --pin R... print the pin entry for each repo at its current main (adding a pin, or re-deriving one). --bump R rewrite repo R's entry in --pins to the current main, for the tracked files only; a tracked file that vanished upstream is refused rather than dropped silently. --out FILE where --check writes its report (default pin-report.md). + --json FILE with --check: also write the sweep as JSON, which is what --pr-body --from reads. + --from FILE with --pr-body: read the sweep from that JSON instead of re-checking every pin. --self-test run against fixtures, no network. Exit codes: 0 nothing moved, 1 a pin moved (or --check found nothing readable), 2 usage, @@ -550,6 +554,25 @@ def fake_tree_missing(repo, rev="main"): expect("--bump left the pin alone", load(path)["pins"]["fake/moved"]["commit"], "new") tree = fake_tree + + # one sweep, N PR bodies: the daily step must not re-read 147 repos per repo it opens a PR for + import contextlib + import io as _io + sweep = os.path.join(d, "sweep.json") + head, tree, commits_since = fake_head_new, fake_tree, fake_commits + expect("--check --json writes the sweep", + main(["--check", "--pins", path, "--out", os.path.join(d, "r.md"), "--json", sweep]), 1) + + def no_read(*a, **k): + raise AssertionError("--pr-body --from must not read the Hub") + + head, tree = no_read, no_read + body = _io.StringIO() + with contextlib.redirect_stdout(body): + body_rc = main(["--pr-body", "fake/moved", "--from", sweep]) + expect("--pr-body --from needs no read, and succeeds", body_rc, 0) + expect("...and prints that repo's body", "fake/moved" in body.getvalue(), True) + head, tree = fake_head_new, fake_tree expect("--bump takes an intact pin", main(["--bump", "fake/moved", "--pins", path]), 0) expect("--bump moved the commit", load(path)["pins"]["fake/moved"]["commit"], "new") expect("--bump took upstream's oid", load(path)["pins"]["fake/moved"]["files"]["tpl.gguf"]["oid"][:4] is not None, True) @@ -592,6 +615,8 @@ def main(argv: list[str]) -> int: ap.add_argument("--bump", metavar="REPO") ap.add_argument("--pins", default=PINS) ap.add_argument("--out", default="pin-report.md") + ap.add_argument("--json", metavar="FILE") + ap.add_argument("--from", dest="src", metavar="FILE") ap.add_argument("--no-template", action="store_true") ap.add_argument("--self-test", action="store_true") a = ap.parse_args(argv) @@ -612,6 +637,20 @@ def main(argv: list[str]) -> int: ap.print_help() return 2 + if a.pr_body and a.src: + # The body for one repo out of a sweep that already happened: the daily step opens N PRs for one + # check, and re-checking 147 repos per PR was 78 s of Hub reads each time. + try: + with open(a.src, encoding="utf-8") as f: + swept = json.load(f)["changes"] + except (OSError, KeyError, ValueError) as e: + sys.exit(f"pin_watch: could not read {a.src}: {e}") + hit = next((c for c in swept if c["repo"] == a.pr_body), None) + if hit is None: + sys.exit(f"pin_watch: {a.pr_body} did not move (see {a.src})") + print(pr_body(hit)) + return 0 + doc = load(a.pins) if a.bump: pin = doc["pins"].get(a.bump) or sys.exit(f"pin_watch: {a.bump} is not pinned") @@ -639,12 +678,17 @@ def main(argv: list[str]) -> int: md = report(changes, notes, len(doc["pins"]), not a.no_template) with open(a.out, "w", encoding="utf-8") as f: f.write(md) + if a.json: + with open(a.json, "w", encoding="utf-8") as f: + json.dump({"date": datetime.date.today().isoformat(), "pins": len(doc["pins"]), + "changes": changes, "notes": notes}, f, indent=1) + f.write("\n") if a.pr_body: hit = next((c for c in changes if c["repo"] == a.pr_body), None) if hit is None: sys.exit(f"pin_watch: {a.pr_body} did not move (see {a.out})") print(pr_body(hit)) - return 1 + return 0 print(md) if notes and len(notes) == len(doc["pins"]): # nothing was read: a failed run, not a clean one. The workflow fails on any code outside 0/1. From dca4ad309f9c4e7a84e1939642b88b9fc4e855e0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 18:45:32 -0400 Subject: [PATCH 4/4] pin-watch: a repo the token cannot see is a permission note, not a clean pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First production run of the workflow (workflow_dispatch on main, run 42123604191): token present, checkout and the sweep worked, and the bump step died exactly where #1181 says it does (`line 23: unexpected EOF while looking for matching "'`). The sweep's first line was worth more than the crash: # Model pins: 145 of 147 checked, 2 not checked, 1 moved (2026-11-05) Two pins are invisible to the workflow's HF_TOKEN — `bong-water-water-bong/ZAYA1-8B-Q4_K_M-GGUF` and `bong-water-water-bong/ZAYA1-8B-QLoRA-GGUF` answer 401 anonymously and 200 with the box's token, i.e. they are private. A 401 is neither transient nor "gone": no token change can happen inside the run, so the honest thing is to say what it means and let a human fix the token. - `check()` reports 401/403 as "the token cannot see this repo (401) — a private or gated repo needs a token with access", still counted in the report's first line (so it can never print an all-clear over it). - the workflow annotates the run with that first line when anything was not checked: visible on the run list without failing the PR flow, which a secret-scope problem would only do every day. - docs/model-pins.md says what a 401 means and that the count is the contract. --self-test 41/41, including a 401 on one repo that must not hide the other repo's move (exit 1, first line '1 of 2 checked, 1 not checked'). --- .github/workflows/pin-watch.yml | 4 ++++ docs/model-pins.md | 5 +++++ tools/pin_watch.py | 22 ++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/.github/workflows/pin-watch.yml b/.github/workflows/pin-watch.yml index 3eb880538..5fc7b0993 100644 --- a/.github/workflows/pin-watch.yml +++ b/.github/workflows/pin-watch.yml @@ -65,6 +65,10 @@ jobs: python3 tools/pin_watch.py --check --out pin-report.md --json pin-report.json || rc=$? if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then exit "$rc"; fi echo "moved=$(grep -c '^## `' pin-report.md || true)" >> "$GITHUB_OUTPUT" + # a pin nobody could read is not a clean pin: surface it on the run, keep the PR flow going + if head -1 pin-report.md | grep -q 'not checked'; then + echo "::warning::$(head -1 pin-report.md) — see **Not checked** in the summary" + fi cat pin-report.md >> "$GITHUB_STEP_SUMMARY" - name: Open one pin bump per changed repo diff --git a/docs/model-pins.md b/docs/model-pins.md index ce5d8a734..10f520cf0 100644 --- a/docs/model-pins.md +++ b/docs/model-pins.md @@ -100,6 +100,11 @@ or a scoreboard row, each of which is its own reviewed change. So: `--bump` refuses a pin whose tracked file vanished upstream instead of dropping it: a file that is gone is a decision (re-take it under its new name, or drop the pin), not a diff. +A repo the token cannot see (`401`/`403`) is reported as that, not as a clean pin: the daily run absorbs the +`HF_TOKEN` secret, and a private or gated repo stays unverified until that token can read it. The report's +first line counts what was read — `145 of 147 checked, 2 not checked` — and the workflow annotates the run +with it, so "nobody could check this" never looks like "nothing moved". + ## Commands ```bash diff --git a/tools/pin_watch.py b/tools/pin_watch.py index b461c1ccc..2c08749cd 100644 --- a/tools/pin_watch.py +++ b/tools/pin_watch.py @@ -281,6 +281,12 @@ def check(pins: dict, want_template: bool) -> tuple[list[dict], list[str]]: now = head(repo) at = {e["path"]: e for e in tree(repo, now) if e.get("type") == "file"} except urllib.error.HTTPError as e: + if e.code in (401, 403): + # not transient and not "gone": the token cannot see it, so nobody checks this pin until a + # token that can read the repo is in place (a private or gated repo). + notes.append(f"{repo}: the token cannot see this repo ({e.code}) — a private or gated repo " + f"needs a token with access; the pin was not checked") + continue if e.code == 404: # "the upstream is gone" is the answer, not a failed read: a delete, a rename and a # privatisation all look like this, and none of them may pass as a clean day. @@ -595,6 +601,22 @@ def fake_head_404(repo): # a run that could read nothing is exit 2 (the workflow fails on anything outside 0/1) and its # first line counts what it read + def fake_head_one_401(repo): + if repo == "fake/gone": + raise urllib.error.HTTPError("https://huggingface.co/api/models/" + repo, 401, "Unauthorized", {}, None) + return "new" + + path401 = os.path.join(d, "pins401.json") + save(path401, {"pins": {"fake/moved": dict(stale), "fake/gone": dict(stale)}}) + head = fake_head_one_401 + report_401 = os.path.join(d, "unauthorised.md") + expect("a 401 on one repo does not hide the other's move", + main(["--check", "--pins", path401, "--out", report_401]), 1) + unread = open(report_401, encoding="utf-8").read() + expect("...it is a permission note, not a clean pin", "the token cannot see this repo (401)" in unread, True) + expect("...and the first line counts it", "1 of 2 checked, 1 not checked" in unread.splitlines()[0], True) + head = fake_head_new + def fake_head_flaky(repo): raise RuntimeError("HTTP Error 429: Too Many Requests")