Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 22 additions & 7 deletions .github/workflows/pin-watch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,14 @@ jobs:
set -uo pipefail
# exit 1 means "a pin moved", which is the normal case this workflow exists for
rc=0
python3 tools/pin_watch.py --check --out pin-report.md || rc=$?
# the JSON beside the report is the same sweep, so the PR step below does not repeat it
python3 tools/pin_watch.py --check --out pin-report.md --json pin-report.json || rc=$?
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then exit "$rc"; fi
echo "moved=$(grep -c '^## `' pin-report.md || true)" >> "$GITHUB_OUTPUT"
# a pin nobody could read is not a clean pin: surface it on the run, keep the PR flow going
if head -1 pin-report.md | grep -q 'not checked'; then
echo "::warning::$(head -1 pin-report.md) — see **Not checked** in the summary"
fi
cat pin-report.md >> "$GITHUB_STEP_SUMMARY"

- name: Open one pin bump per changed repo
Expand All @@ -82,17 +87,27 @@ jobs:
if git ls-remote --exit-code origin "refs/heads/$branch" > /dev/null 2>&1; then
echo "$branch already exists"; continue
fi
git switch -c "$branch" main --quiet
if ! python3 tools/pin_watch.py --bump "$repo"; then
# a tracked file vanished upstream: the report says so, and a human decides
echo "::warning::$repo could not be bumped automatically (see the report)"
# -C, not -c: a bump that failed earlier in this run left its branch behind, and a
# "branch already exists" abort here would leave every later repo unbumped
git switch -C "$branch" main --quiet
rc=0
python3 tools/pin_watch.py --bump "$repo" || rc=$?
if [ "$rc" -eq 4 ]; then
# gone upstream, or a tracked file did: the report says so, and a human decides
echo "::warning::$repo needs a human decision (see the report); no bump opened"
git checkout -- models/pins.json 2>/dev/null || true
git switch main --quiet
continue
elif [ "$rc" -ne 0 ]; then
# anything else is an unread upstream, which is not a clean day
echo "::error::$repo could not be bumped (rc=$rc): this run is not a clean one"
exit 1
fi
python3 tools/pin_watch.py --pr-body "$repo" > body.md || true
# no `|| true`: a body that could not be produced is a broken report, not an empty PR
python3 tools/pin_watch.py --pr-body "$repo" --from pin-report.json > body.md
newsha=$(python3 -c 'import json,sys; print(json.load(open("models/pins.json"))["pins"][sys.argv[1]]["commit"][:8])' "$repo")
git add models/pins.json
git commit -q -m "pin: $repo at $newsha""
git commit -q -m "pin: $repo at $newsha"
git push -q origin "$branch"
gh pr create --base main --head "$branch" --title "Pin: $repo" --body-file body.md
git switch main --quiet
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -30,3 +30,7 @@ Testing/
# the HRX toolchain the build fetches (scripts/fetch-rocm-sdk.sh, config/hrx-toolchain.config):
# a multi-gigabyte venv of TheRock's ROCm SDK at the pinned version, not part of the source
third_party/toolchain/

# the model-pin sweep writes these where it runs (tools/pin_watch.py --check --json)
pin-report.md
pin-report.json
4 changes: 4 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,10 @@ if(Python3_Interpreter_FOUND)
# pin while going red every day unnoticed. The step is driven against a stubbed gh, with and
# without an open issue.
add_test(NAME bump_workflow_report COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/bump_workflow_report_test.py)
# every workflow's `run:` must parse, and the pin bump step must open one PR per moved repo while a
# repo that went upstream stays a human decision: the step shipped with an unterminated quote that
# nothing executed until a pin moved, i.e. until its first scheduled run (engine #1148)
add_test(NAME workflow_shell COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/workflow_shell_test.py)

# a Q4NX directory's chunk layout (engine #587): synthetic containers, so the check a
# published FastFlowLM model goes through is proved to fire without the model at hand.
Expand Down
9 changes: 8 additions & 1 deletion docs/model-pins.md
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,19 @@ or a scoreboard row, each of which is its own reviewed change. So:
`--bump` refuses a pin whose tracked file vanished upstream instead of dropping it: a file that is gone is
a decision (re-take it under its new name, or drop the pin), not a diff.

A repo the token cannot see (`401`/`403`) is reported as that, not as a clean pin: the daily run absorbs the
`HF_TOKEN` secret, and a private or gated repo stays unverified until that token can read it. The report's
first line counts what was read — `145 of 147 checked, 2 not checked` — and the workflow annotates the run
with it, so "nobody could check this" never looks like "nothing moved".

## Commands

```bash
tools/pin_watch.py --check # the daily comparison; writes pin-report.md, exit 0/1/2 (above)
tools/pin_watch.py --check --no-template # same, without the template range reads
tools/pin_watch.py --pr-body REPO # the bump PR body for one moved repo
tools/pin_watch.py --check --json sweep.json # ...and the same sweep as JSON
tools/pin_watch.py --pr-body REPO --from sweep.json # that repo's PR body, from that sweep: no second read
tools/pin_watch.py --pr-body REPO # without --from it repeats the sweep (one full read per repo)
tools/pin_watch.py --pin REPO [REPO...] # print an entry for a repo at its current main (adding a pin)
tools/pin_watch.py --bump REPO # rewrite one entry in models/pins.json (a reviewed change)
tools/pin_watch.py --self-test # fixtures, no network
Expand Down
210 changes: 210 additions & 0 deletions tests/workflow_shell_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A workflow's shell must parse, and the pin bump step must open one PR per moved repo.

pin-watch.yml's "Open one pin bump per changed repo" step was committed with an unterminated quote
(`git commit -m "pin: $repo at $newsha""`). Nothing executed that step until a pin actually moved, so a
syntax error that kills the whole step - and with it every bump PR - sat on main looking green: the
scheduled run at 05:43 UTC would have been its first reader.

Two guarantees, both offline (CI installs no PyYAML, so the blocks are lifted out of the text):

* every `run:` in every workflow parses under `bash -n` (the class, not just this instance);
* the pin bump step, driven against a local bare origin, a stubbed `gh` and a stubbed `python3`, opens
exactly one PR per moved repo, leaves a repo that went upstream to a human, and fails the run when an
upstream could not be read at all - an unread upstream is not a clean day.
"""
import json
import os
import re
import subprocess
import sys
import tempfile

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
WORKFLOWS = os.path.join(ROOT, ".github", "workflows")
STEP = "Open one pin bump per changed repo"
failures = []


def check(what, cond, detail=""):
if cond:
print(f"ok {what}")
else:
print(f"FAIL {what}" + (f": {detail}" if detail else ""))
failures.append(what)


def blocks(text):
"""[(line number, shell)] for every `run: |` block and every inline `run:` scalar."""
lines = text.splitlines()
out, i = [], 0
while i < len(lines):
m = re.match(r"^(\s*)(?:- )?run: (\||>[-+]?|)\s*$", lines[i])
inline = re.match(r"^(\s*)(?:- )?run: (?!\||>)(\S.*)$", lines[i])
if m:
indent, body, j = len(m.group(1)), [], i + 1
while j < len(lines) and (not lines[j].strip() or len(lines[j]) - len(lines[j].lstrip()) > indent):
body.append(lines[j])
j += 1
while body and not body[-1].strip():
body.pop()
cut = min((len(b) - len(b.lstrip()) for b in body if b.strip()), default=0)
out.append((i + 1, "\n".join(b[cut:] for b in body) + "\n"))
i = j
elif inline:
out.append((i + 1, inline.group(2)))
i += 1
else:
i += 1
return out


def step_script(text, name):
"""The de-indented shell of the named step, or None."""
lines = text.splitlines()
at = next((i for i, l in enumerate(lines) if f"- name: {name}" in l), None)
if at is None:
return None
found = blocks("\n".join(lines[at:]))
return found[0][1] if found else None


def sandbox(tmp, report):
"""A bare origin plus a working clone carrying a pins file and the report the check step writes."""
origin, work = os.path.join(tmp, "origin.git"), os.path.join(tmp, "work")
subprocess.run(["git", "init", "-q", "--bare", origin], check=True)
subprocess.run(["git", "clone", "-q", origin, work], check=True, capture_output=True)
for cmd in (["config", "user.email", "t@example.com"], ["config", "user.name", "t"]):
subprocess.run(["git", "-C", work] + cmd, check=True)
os.makedirs(os.path.join(work, "models"))
os.makedirs(os.path.join(work, "tools"))
pins = {"version": 1, "pins": {}}
for repo in ("ok/one", "ok/two", "human/gone", "broken/x"):
pins["pins"][repo] = {"commit": "a" * 40, "pinned": "2026-11-02", "used_for": ["t"], "files": {}}
with open(os.path.join(work, "models", "pins.json"), "w") as fh:
json.dump(pins, fh, indent=1)
open(os.path.join(work, "tools", "pin_watch.py"), "w").close()
with open(os.path.join(work, "pin-report.md"), "w") as fh:
fh.write("# Model pins: 4 of 4 moved (2026-11-05)\n\n")
for c in report:
fh.write(f"## `{c['repo']}`\n\n{c['body']}\n")
with open(os.path.join(work, "pin-report.json"), "w") as fh:
json.dump({"date": "2026-11-05", "changes": report}, fh)
subprocess.run(["git", "-C", work, "add", "-A"], check=True)
subprocess.run(["git", "-C", work, "commit", "-qm", "init"], check=True)
subprocess.run(["git", "-C", work, "branch", "-M", "main"], check=True)
subprocess.run(["git", "-C", work, "push", "-q", "-u", "origin", "main"], check=True)

stub = os.path.join(tmp, "stub")
os.makedirs(stub)
log = os.path.join(tmp, "gh.log")
with open(os.path.join(stub, "gh"), "w") as fh:
fh.write(f'#!/bin/sh\nprintf "%s\\n" "$*" >> "{log}"\necho https://example/pr/1\n')
# `python3` stands in for the tool's two network calls; the step's own `python3 -c` still runs for real
with open(os.path.join(stub, "python3"), "w") as fh:
fh.write(f'''#!/bin/sh
REAL={sys.executable}
if [ "$1" = tools/pin_watch.py ] && [ "$2" = --bump ]; then
case "$3" in
human/gone) echo "pin_watch: $3: gone upstream (404)" >&2; exit 4 ;;
broken/x) echo "pin_watch: $3: could not read the Hub" >&2; exit 1 ;;
esac
"$REAL" -c 'import json,sys; p="models/pins.json"; d=json.load(open(p)); d["pins"][sys.argv[1]]["commit"]="b"*40; json.dump(d,open(p,"w"),indent=1)' "$3" || exit 1
exit 0
fi
if [ "$1" = tools/pin_watch.py ] && [ "$2" = --pr-body ]; then
# the JSON beside the report is the sweep this must read, so a missing --from fails the step here
if [ "$4" != --from ] || [ ! -s "$5" ]; then echo "pr-body without the sweep: $*" >&2; exit 7; fi
echo "body for $3"; exit 0
fi
exec "$REAL" "$@"
''')
for f in ("gh", "python3"):
os.chmod(os.path.join(stub, f), 0o700)
return work, stub, log


def drive(script, work, stub, log):
env = dict(os.environ, PATH=stub + os.pathsep + os.environ.get("PATH", ""), GH_TOKEN="test")
p = subprocess.run(["bash", "-c", script], cwd=work, env=env, capture_output=True, text=True)
calls = open(log).read().splitlines() if os.path.exists(log) else []
heads = subprocess.run(["git", "-C", work, "ls-remote", "--heads", "origin"],
capture_output=True, text=True).stdout.split()
return p, [c for c in calls if c.strip()], heads


# 1. every workflow's shell parses - the class the pin-watch step was caught by
for name in sorted(os.listdir(WORKFLOWS)):
if not name.endswith(".yml"):
continue
text = open(os.path.join(WORKFLOWS, name)).read()
found = blocks(text)
bad = []
for line, shell in found:
p = subprocess.run(["bash", "-n"], input=shell, text=True, capture_output=True)
if p.returncode != 0:
bad.append(f"line {line}: {p.stderr.strip().splitlines()[-1]}")
check(f"{name}: {len(found)} run block(s) parse", not bad, "; ".join(bad)[:200])

# 2. the pin bump step: one PR per moved repo, a gone repo left to a human, an unread one fatal
text = open(os.path.join(WORKFLOWS, "pin-watch.yml")).read()
script = step_script(text, STEP)
check("pin-watch: has the pin bump step", script is not None)
if script:
p = subprocess.run(["bash", "-n"], input=script, text=True, capture_output=True)
check("pin-watch: the pin bump step's shell parses", p.returncode == 0, p.stderr.strip()[:200])

report = [{"repo": "ok/one", "body": "`a` -> `b`"},
{"repo": "human/gone", "body": "**Gone upstream** — the Hub answers `404`."},
{"repo": "ok/two", "body": "`a` -> `b`"}]
with tempfile.TemporaryDirectory() as tmp:
work, stub, log = sandbox(tmp, report)
p, calls, heads = drive(script, work, stub, log)
creates = [c for c in calls if c.startswith("pr create")]
check("pin-watch: a gone repo does not fail the run", p.returncode == 0, p.stderr.strip()[-200:])
check("pin-watch: one 'gh pr create' per moved repo", len(creates) == 2, str(calls))
check("pin-watch: the gone repo is warned about, not bumped",
"human/gone needs a human decision" in p.stdout + p.stderr)
branches = sorted(h.split("refs/heads/")[-1] for h in heads if "refs/heads/pin-watch/" in h)
check("pin-watch: one branch per moved repo, none for the gone one", len(branches) == 2, str(branches))
for repo in ("ok/one", "ok/two"):
slug = repo.replace("/", "-").replace(".", "-")
branch = next((b for b in branches if b.startswith(f"pin-watch/{slug}-")), None)
check(f"pin-watch: {repo} has a pushed branch at main", branch is not None, str(branches))
if branch is None:
continue
check(f"pin-watch: {repo}'s PR names that branch",
any(c.startswith("pr create") and f"--head {branch}" in c for c in creates), str(creates))
msg = subprocess.run(["git", "-C", work, "log", "-1", "--format=%s", branch],
capture_output=True, text=True).stdout.strip()
check(f"pin-watch: {repo}'s commit says what it pins", msg == f"pin: {repo} at bbbbbbbb", msg)
files = subprocess.run(["git", "-C", work, "show", "--name-only", "--format=", branch],
capture_output=True, text=True).stdout.split()
check(f"pin-watch: {repo}'s commit touches only the pins file", files == ["models/pins.json"], str(files))

report = [{"repo": "broken/x", "body": "`a` -> `b`"}]
with tempfile.TemporaryDirectory() as tmp:
work, stub, log = sandbox(tmp, report)
p, calls, heads = drive(script, work, stub, log)
check("pin-watch: an unread upstream fails the run instead of a green no-PR day",
p.returncode != 0 and "could not be bumped" in p.stdout + p.stderr, f"rc={p.returncode}")
check("pin-watch: and opens nothing", not [c for c in calls if c.startswith("pr create")], str(calls))

if failures:
print(f"\n{len(failures)} check(s) failed")
sys.exit(1)
print("\nall checks passed")
Loading
Loading