diff --git a/CMakeLists.txt b/CMakeLists.txt index bc5cf6b11..5193b3705 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -241,6 +241,11 @@ add_test(NAME config_pin_check COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/conf WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) add_test(NAME config_pin_check_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/config_pin_check.py --self-test WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) +# Gantt "Kernel 7.4 readiness": the inputs written inline in workflows and build scripts (a container image, a +# `uses:` ref, a raw download, a pip requirement) name a commit or a digest. CI's `ubuntu:26.04` changed digest between +# 2026-11-09 and 2026-11-13 with no change in this tree. Offline, reads the tree only. +add_test(NAME moving_ref_check COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/moving_ref_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_test(NAME moving_ref_check_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/moving_ref_check.py --self-test) # engine#2183: the lost-update class has a second medium. `registration_loss_check` covers what a branch removes # from the BUILD; this covers what it removes from the DOCS -- a rebase that resolved `docs/npu.md` by taking the # branch's side dropped a section that had merged that morning (#2314). Offline, git-only, no build input. diff --git a/docs/model-pins.md b/docs/model-pins.md index 60a8b4e63..be21b3a6f 100644 --- a/docs/model-pins.md +++ b/docs/model-pins.md @@ -145,6 +145,28 @@ makes the run's first line say so, and adding it is `--pin` plus a `used_for` th A repo of `.1bp` packs is not pinned here, and a private repo never is (next section) — both are filtered out, so the census cannot go permanently red over something this file must not contain. +## Build inputs + +The model pins above are one kind of input. A build or CI run also takes inputs written inline in workflows and +scripts, and each of those is named by a commit or a digest too, so a release is a pin bump and not a scramble: + +| input | pinned where | example | +|---|---|---| +| CI container | `.github/workflows/ci.yml` `image:` | `ubuntu:26.04@sha256:065aa862…` (the tag moved between 2026-11-09 and 2026-11-13). **Owed**: a workflow edit needs the `workflow` token scope, so it is baselined in `tools/ci/moving_ref_baseline.txt` until the owner applies it | +| CI Python packages | the `pip install` line | `numpy==2.5.4 gguf==0.19.0` (**owed**, same reason) | +| GitHub Actions | every `uses:` | a 40-hex commit | +| package-build containers | `scripts/pack-distro.sh` | `fedora:45@sha256:…`, `archlinux:base-20261108.0.623074@sha256:…` | +| Puppy appliance | [`os/v3/puppy/pin.txt`](../os/v3/puppy/pin.txt) | the woof-CE commit, the build container digest, `vercmp.c` by commit + sha256 | + +[`tools/ci/moving_ref_check.py`](../tools/ci/moving_ref_check.py) (ctest `moving_ref_check`) fails a tree that names +a branch, a tag or no version in any of these places, except the rows listed in +[`tools/ci/moving_ref_baseline.txt`](../tools/ci/moving_ref_baseline.txt). That baseline only shrinks: a row that no +longer matches anything fails the check too. A read that is moving on purpose carries a +`moving-ref: ` comment on the line or the line above it, and the check prints that reason. Two kinds of read +are marked this way: the bump bots, which read the newest release only to propose a pin, and `scripts/pack-proof.sh`, +whose point is "installs on this distro as users pull it today". A new image or version is a reviewed edit of the +line that names it. + ## What is not pinned, and why - `base_model` on a card is provenance, not a dependency: what we took from a vendor repo is the file we diff --git a/os/v3/puppy/README.md b/os/v3/puppy/README.md index 8f74d6abf..a065d0a62 100644 --- a/os/v3/puppy/README.md +++ b/os/v3/puppy/README.md @@ -32,6 +32,7 @@ not with this installer. The Ubuntu design in [docs/os-v3.md](../../../docs/os-v | `sb-vars.sh` | OVMF's variable store with our certificate enrolled in `db`, so a QEMU boot verifies the appliance the way the board does | | `mk-huge-kernel.sh` | our kernel as woof-CE's kernel package (`huge-.tar.*`: `vmlinuz-` + a zdrv SFS with the module tree, the firmware and a marker). **Refuses an unsigned kernel image** | | `build.sh` | woof-CE's CI recipe in a throwaway container, with our kernel and the build root on a host filesystem. Host side only: it resolves the inputs, then runs `container-build.sh` | +| `pin.txt` | the inputs by commit or digest: the woof-CE commit, the build container (`ubuntu:24.04@sha256:…`), and `vercmp.c` by initrd_progs commit + sha256. `build.sh` **refuses a woof-CE clone at any other commit** (`WOOF_CE_UNPINNED=1` builds anyway, with a warning that the image is not reproducible from the tree); `container-build.sh` refuses a `vercmp.c` whose sha256 differs | | `container-build.sh` | what runs **inside** the container (merge2out → kernel into `kernel-kit/output` → 0setup → 1download → 2createpackages → 3builddistro, then the embed and the artifact checks). Its own file since cycle 14: as a 300-line single-quoted `bash -c '…'` string inside `build.sh`, one apostrophe in a comment ended the quote and bash reported a syntax error a hundred lines away — five builds in this lane — and shellcheck could not check it at all | | `g2-probe.sh` | gate G2: boot the built ISO headless in QEMU and keep the init's transcript, so "it boots on our kernel" is a measurement | | `g3-probe.sh` | gate G3, artifact half: the ISO carries our kernel, **that kernel image is signed**, its zdrv carries the module tree, and `amdxdna`/`ec_su_axb35` are built for that kernel and signed. `--cert ` requires the image's signature to be the enrolled MOK's; `--require-mok` for the release gate | @@ -113,6 +114,8 @@ os/v3/puppy/mk-huge-kernel.sh os/v3/puppy/build.sh resolute64 [woof-CE-dir] # (woof-CE defaults to $HOME/wt/woof-CE - not /tmp, which a reboot empties; the 4th argument is the # clone, and the build copies it into the build root itself) +# The clone must be at the woof_ce_commit in os/v3/puppy/pin.txt: +# git clone https://github.com/puppylinux-woof-CE/woof-CE ~/wt/woof-CE-pin && git -C ~/wt/woof-CE-pin checkout # A new BUILD_ROOT downloads 2.6 GB again, and one mirror hiccup over a thousand files fails the # build (measured: one package, all four mirrors, and the URL was fine minutes later). If the box has # built before, seed the cache instead: sudo cp -a /local-repositories / diff --git a/os/v3/puppy/build.sh b/os/v3/puppy/build.sh index 2e12fdaf6..0ff364cfb 100755 --- a/os/v3/puppy/build.sh +++ b/os/v3/puppy/build.sh @@ -98,6 +98,21 @@ case "$woof_real/" in exit 1 ;; esac kernel=$(cd "$kernel" && pwd) +# The pinned inputs (os/v3/puppy/pin.txt): the woof-CE commit, the build container by digest, vercmp.c by commit. +pinfile="$(cd "$(dirname "$0")" && pwd)/pin.txt" +pin() { sed -n "s/^$1=//p" "$pinfile" | head -n 1; } +woof_pin=$(pin woof_ce_commit) build_image=$(pin build_image) +[ -n "$woof_pin" ] && [ -n "$build_image" ] || { echo "build: REFUSED: $pinfile names no woof_ce_commit / build_image" >&2; exit 1; } +woof_head=$(git -C "$woof" rev-parse HEAD 2>/dev/null || echo "no git metadata") +if [ "$woof_head" != "$woof_pin" ]; then + if [ "${WOOF_CE_UNPINNED:-0}" = 1 ]; then + echo "build: WARNING: woof-CE at $woof is $woof_head, the pin is $woof_pin (WOOF_CE_UNPINNED=1: building anyway; this image is not reproducible from the tree)" >&2 + else + printf 'build: REFUSED: woof-CE at %s is %s, os/v3/puppy/pin.txt pins %s.\n' "$woof" "$woof_head" "$woof_pin" >&2 + printf 'build: git -C %s fetch origin && git -C %s checkout %s (or WOOF_CE_UNPINNED=1 for a deliberate experiment)\n' "$woof" "$woof" "$woof_pin" >&2 + exit 1 + fi +fi ls "$kernel"/huge-*.tar.* >/dev/null 2>&1 || { echo "build: no huge-*.tar.* in $kernel (run mk-huge-kernel.sh first)" >&2; exit 1; } mkdir -p "$out"; out=$(cd "$out" && pwd) @@ -161,7 +176,7 @@ if [ -n "$tui_bin" ]; then echo "build: console TUI $tui_bin ($(sha256sum "$tui_bin" | cut -c1-16))" fi -echo "build: woof-CE $woof, kernel $kernel, out $out, build root $build_root ($mnt_opts)" +echo "build: woof-CE $woof ($woof_head), image $build_image, kernel $kernel, out $out, build root $build_root ($mnt_opts)" # The container side is its own file (os/v3/puppy/container-build.sh, mounted at /recipe) since 2026-11-02: # it was a 300-line single-quoted `bash -c '...'` string here, where one apostrophe in a comment ended the # quote and bash reported a syntax error a hundred lines away - five builds in this lane, two of them in @@ -179,4 +194,4 @@ exec docker run --rm --privileged --memory=12g --cpus=4 \ ${tui_bin:+-v "$tui_bin":/console-tui:ro} \ ${tui_bin:+-e CONSOLE_TUI_BIN=/console-tui} \ --device /dev/loop-control:/dev/loop-control \ - -e COMPAT="$compat" ubuntu:24.04 bash /recipe/container-build.sh + -e COMPAT="$compat" "$build_image" bash /recipe/container-build.sh diff --git a/os/v3/puppy/container-build.sh b/os/v3/puppy/container-build.sh index e61ead490..dbc71290e 100644 --- a/os/v3/puppy/container-build.sh +++ b/os/v3/puppy/container-build.sh @@ -87,7 +87,13 @@ bash -n woof-code/3builddistro || { echo "build: REFUSED: the patched 3builddist echo " ok: 3builddistro normalises rootfs-complete and devx before packing, keeping their setuid bits" mkdir -p /work/local-repositories petbuild-sources petbuild-cache petbuild-output [ -e ../local-repositories ] || ln -s /work/local-repositories ../local-repositories -[ -f local-repositories/vercmp ] || curl -fsSL https://raw.githubusercontent.com/puppylinux-woof-CE/initrd_progs/master/pkg/w_apps_static/w_apps/vercmp.c | gcc -x c -o local-repositories/vercmp - +# vercmp.c at the commit and sha256 os/v3/puppy/pin.txt names (it came from initrd_progs `master` before 2026-11-13) +if [ ! -f local-repositories/vercmp ]; then + vercmp_url=$(sed -n 's/^vercmp_url=//p' /recipe/pin.txt) vercmp_sha=$(sed -n 's/^vercmp_sha256=//p' /recipe/pin.txt) + curl -fsSL "$vercmp_url" -o /tmp/vercmp.c + echo "$vercmp_sha /tmp/vercmp.c" | sha256sum -c --quiet - || { echo "build: REFUSED: vercmp.c from $vercmp_url is not sha256 $vercmp_sha" >&2; exit 1; } + gcc -x c -o local-repositories/vercmp /tmp/vercmp.c +fi install -m 755 local-repositories/vercmp /usr/local/bin/vercmp install -D -m 644 woof-code/rootfs-skeleton/usr/local/petget/categories.dat /usr/local/petget/categories.dat # initrd-progs packs its initrd with `find . | cpio -o -H newc > ../initrd 2>/dev/null`, so without cpio the diff --git a/os/v3/puppy/pin.txt b/os/v3/puppy/pin.txt new file mode 100644 index 000000000..ff0bec074 --- /dev/null +++ b/os/v3/puppy/pin.txt @@ -0,0 +1,28 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# The inputs the Puppy appliance is built from, each named by something that cannot move. +# build.sh reads this file; a new input is a reviewed change to this file (Refs Gantt "Kernel 7.4 readiness"). +# +# woof-CE: the commit the cycle-17..21 images were built from (2026-10-27, "Update kernel-kit-k6.18.yml"). +# Upstream `testing` was 3 commits past it by 2026-11-13, so a fresh clone no longer builds the same image. +# build.sh refuses a clone at any other commit unless WOOF_CE_UNPINNED=1 is set. +woof_ce_repo=https://github.com/puppylinux-woof-CE/woof-CE +woof_ce_commit=c3a5c6d42b41d424ebeb24e6fc05d31da8f11fac +# The container the build runs in: ubuntu:24.04 by digest (resolved from Docker Hub 2026-11-13). +build_image=ubuntu:24.04@sha256:2198e4c65076e9cb0b3d695219df667c0e3b0072da64ca3c2648aca17dcecaf5 +# vercmp.c from initrd_progs. It used to come from `master`; this is the commit master pointed at on 2026-11-13 +# (2025-12-27). The sha256 is checked before the file is compiled. +vercmp_url=https://raw.githubusercontent.com/puppylinux-woof-CE/initrd_progs/176c2df25840dbe1bb34fa9f71e9a7b6904ec9e7/pkg/w_apps_static/w_apps/vercmp.c +vercmp_sha256=271fe14a033714ccdc48b186527b0ee78b2c5e4b775aebe7f9a9a1d1ecd24a9e diff --git a/scripts/pack-distro.sh b/scripts/pack-distro.sh index 03795f7a7..779812bf5 100755 --- a/scripts/pack-distro.sh +++ b/scripts/pack-distro.sh @@ -189,7 +189,7 @@ SH docker run --rm -v "$work:/build" -w /build -e HOSTUID="$(id -u)" -e HOSTGID="$(id -g)" \ -e VER="$ver" -e REL="$rel" -e SPEC="$(basename "$spec")" \ -e CHLOGDATE="$(LC_ALL=C date '+%a %b %d %Y')" \ - fedora:latest bash /build/in-container.sh > "$work/rpmbuild.log" 2>&1 || + fedora:45@sha256:a7d72ba2745b100a3351087472e5ee7b2450af7cf331e7673007e3817612839c bash /build/in-container.sh > "$work/rpmbuild.log" 2>&1 || { tail -30 "$work/rpmbuild.log" >&2; exit 1; } built=$(find "$work/rpmbuild/RPMS" -name '*.rpm' -type f | head -1) [ -n "$built" ] || { echo "pack-distro.sh: rpmbuild produced no package" >&2; exit 1; } @@ -220,7 +220,7 @@ su builder -c 'cd /build && makepkg -f --nodeps --noconfirm' SH docker run --rm -v "$work:/build" -w /build -e HOSTUID="$(id -u)" -e HOSTGID="$(id -g)" \ -e PKGVER="$ver" -e PKGREL="$rel" \ - archlinux:latest bash /build/in-container.sh > "$work/makepkg.log" 2>&1 || + archlinux:base-20261108.0.623074@sha256:d4594bb2300cba11f41c2f27923eb56f63d9988909c402f9c95517d1e362c899 bash /build/in-container.sh > "$work/makepkg.log" 2>&1 || { tail -30 "$work/makepkg.log" >&2; exit 1; } built=$(find "$work" -maxdepth 1 -name '*.pkg.tar.zst' -type f | head -1) [ -n "$built" ] || { echo "pack-distro.sh: makepkg produced no package" >&2; exit 1; } diff --git a/scripts/pack-proof.sh b/scripts/pack-proof.sh index ed4059ea0..09ae49d54 100755 --- a/scripts/pack-proof.sh +++ b/scripts/pack-proof.sh @@ -103,6 +103,7 @@ apt-get remove -y -qq 1bit-engine >/dev/null 2>&1 || { echo 'apt-get remove fail [ ! -e /opt/1bit/engine/1bit ] || { echo 'the payload survived the removal'; exit 1; } " +# moving-ref: the proof is that the package installs on this distro as a user pulls it today, so the newest image is the point run_row "rpm, fedora" fedora:latest " dnf -y -q install /pkg/$rpm >/dev/null 2>&1 || { echo 'dnf install failed'; exit 1; } $body @@ -110,6 +111,7 @@ dnf -y -q remove 1bit-engine >/dev/null 2>&1 || { echo 'dnf remove failed'; exit [ ! -e /opt/1bit/engine/1bit ] || { echo 'the payload survived the removal'; exit 1; } " +# moving-ref: the proof is that the package installs on this distro as a user pulls it today, so the newest image is the point run_row "rpm, opensuse tumbleweed" opensuse/tumbleweed:latest " zypper --non-interactive --gpg-auto-import-keys install --allow-unsigned-rpm /pkg/$rpm >/dev/null 2>&1 || { echo 'zypper install failed'; exit 1; } @@ -118,6 +120,7 @@ zypper --non-interactive remove 1bit-engine >/dev/null 2>&1 || { echo 'zypper re [ ! -e /opt/1bit/engine/1bit ] || { echo 'the payload survived the removal'; exit 1; } " +# moving-ref: the proof is that the package installs on this distro as a user pulls it today, so the newest image is the point run_row "arch, pacman" archlinux:latest " pacman -Sy --noconfirm >/dev/null 2>&1 || { echo 'pacman -Sy failed'; exit 1; } pacman -U --noconfirm /pkg/$arch_pkg >/dev/null 2>&1 || { echo 'pacman -U failed'; exit 1; } diff --git a/tools/ci/moving_ref_baseline.txt b/tools/ci/moving_ref_baseline.txt new file mode 100644 index 000000000..895516287 --- /dev/null +++ b/tools/ci/moving_ref_baseline.txt @@ -0,0 +1,22 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Moving build inputs that are known and owed (tools/ci/moving_ref_check.py). One row per finding: +# pathruletextwhy it waits +# The check fails on a finding not listed here AND on a row that no longer matches, so this file only shrinks. +# All three rows are workflow files: a push that changes .github/workflows needs the `workflow` token scope, which the +# lanes do not have. The fix is the patch named in the PR that added this file; the owner applies it, then deletes these rows. +.github/workflows/ci.yml image ubuntu:26.04 pin to ubuntu:26.04@sha256:065aa8626841f16057b78b027b464cd59c4fac36c204c72dcbf48d66f5fa8a96 (workflow scope) +.github/workflows/ci.yml pip numpy gguf (no ==version) pin numpy==2.5.4 gguf==0.19.0 (workflow scope) +.github/workflows/bump-tokenizers.yml latest releases/latest deliberate (the bump bot proposes a pin): add a moving-ref comment (workflow scope) diff --git a/tools/ci/moving_ref_check.py b/tools/ci/moving_ref_check.py new file mode 100755 index 000000000..14ce274cd --- /dev/null +++ b/tools/ci/moving_ref_check.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""moving_ref_check.py -- a build input names a commit or a digest, never a branch or a tag. + +A release should be a pin bump, not a scramble. That only holds if every input a build or CI run takes is named +by something that cannot move. The other pin checks cover their own files: + - check_pins.py and submodule_pin_check.py cover submodule gitlinks; + - config_pin_check.py covers config/*.config; + - the os/v2 snapshot and the os/v3 ISO sha cover the OS bases. +Nothing looked at the inputs written inline in workflows and build scripts, and two of them had moved: + - CI's `container: ubuntu:26.04` was sha256:6a06111c... on 2026-11-09 and sha256:065aa862... on 2026-11-13, + so CI built on two different images with no change in this tree; + - os/v3/puppy fetched vercmp.c from initrd_progs `master` and built from whatever woof-CE commit the clone + happened to be on. + +Rules. A line breaks one when it names: + uses a GitHub Action `uses: owner/repo@ref` whose ref is not a 40-hex commit; + image a container `image:` with no `@sha256:` digest; + docker a `docker|podman run|create|pull ... name:tag` with no `@sha256:` digest; + raw-ref a raw.githubusercontent.com / codeload / `archive/` URL whose ref is not a 40-hex commit; + latest a `releases/latest` URL, or a `:latest` image; + pip a workflow `pip install` of a requirement with no `==` version. + +A line that reads a moving ref ON PURPOSE carries `moving-ref: ` in a comment on that line or the line above. +The bump bots are an example: they read the tip in order to propose a pin, and do not consume it. The reason is +printed in the report, so an exemption is visible, not silent. + +Scope: .github/workflows/*.yml, scripts/, os/ and tools/ shell scripts, and CMakeLists.txt / cmake/*.cmake. +Probes and tests that talk to 127.0.0.1 are not inputs, and no rule matches them. + +Baseline: tools/ci/moving_ref_baseline.txt lists findings known and owed, one `pathruletextwhy` per +line. A baselined finding is printed and does not fail. A NEW finding fails, and so does a STALE entry (one that no +longer matches anything), so the baseline can only shrink. It exists for one reason: the workflow files can only be +changed by a push with the `workflow` scope, so their fixes wait on the owner. + +Usage: + moving_ref_check.py --check [--root .] [--baseline FILE] + moving_ref_check.py --self-test +Exit: 0 clean, 1 a moving ref (each one is printed as path:line: rule: text), 2 usage, 3 self-test failed. +""" +from __future__ import annotations + +import argparse +import os +import re +import sys +import tempfile + +SHA40 = re.compile(r"^[0-9a-f]{40}$") +PRAGMA = re.compile(r"moving-ref:\s*(\S.*)") + +USES = re.compile(r"^\s*-?\s*uses:\s*([^\s#]+)") +IMAGE = re.compile(r"^\s*image:\s*([^\s#]+)") +# `docker run ... ubuntu:24.04 bash x` -- the first token that looks like name[:tag] after the subcommand's +# options is hard to find in general, so match any registry-shaped `name:tag` on a docker/podman line. +DOCKER = re.compile(r"\b(docker|podman)\s+(run|create|pull)\b") +IMAGE_TOKEN = re.compile(r"(? list[str]: + """The requirements on a `pip install` line that carry no `==` version.""" + bad = [] + skip_next = False + for tok in args.split(): + if tok.startswith("#") or tok in ("|", "&&", ";", "||"): + break + if skip_next: + skip_next = False + continue + if tok in ("-r", "--requirement", "-c", "--constraint", "-e", "--editable", "--index-url", "-i", + "--extra-index-url", "-f", "--find-links", "--target", "-t", "--prefix", "--root"): + skip_next = True + continue + if tok.startswith("-") or tok.startswith((".", "/", "$", '"$', "'")) or "/" in tok: + continue + if "==" not in tok: + bad.append(tok) + return bad + + +def scan_text(path: str, text: str) -> list[tuple[int, str, str, str]]: + """[(line, rule, text, exemption-or-'')] for one file's text.""" + out = [] + raw = text.splitlines() + # A shell command continued with `\` is one command: `docker run ... \` on one line and the image ten lines down + # is still a `docker run` of that image (os/v3/puppy/build.sh is written exactly so). Join each continued line + # into its first line, keep the first line's number, and leave the continuation lines empty so numbers stay put. + lines = list(raw) + i = 0 + while i < len(lines): + j = i + while lines[j].rstrip().endswith("\\") and j + 1 < len(lines) and not lines[j].lstrip().startswith("#"): + j += 1 + if j > i: + lines[i] = " ".join(l.rstrip().rstrip("\\") for l in raw[i:j + 1]) + for k in range(i + 1, j + 1): + lines[k] = "" + i = j + 1 + is_workflow = path.replace(os.sep, "/").startswith(".github/workflows/") + for i, line in enumerate(lines): + if "127.0.0.1" in line or "localhost" in line: + continue + pragma = PRAGMA.search(line) or (PRAGMA.search(lines[i - 1]) if i else None) + why = pragma.group(1).strip() if pragma else "" + code = line.split(" #", 1)[0] if not line.lstrip().startswith("#") else "" + if not code.strip(): + continue + hits = [] + if is_workflow: + m = USES.match(code) + if m and not m.group(1).startswith(("./", "docker://")): + ref = m.group(1).rpartition("@")[2] if "@" in m.group(1) else "" + if not SHA40.match(ref): + hits.append(("uses", m.group(1))) + m = IMAGE.match(code) + if m and "@sha256:" not in m.group(1): + hits.append(("image", m.group(1))) + m = PIP.search(code) + if m: + bad = pip_findings(m.group(1)) + if bad: + hits.append(("pip", " ".join(bad) + " (no ==version)")) + if DOCKER.search(code): + for t in IMAGE_TOKEN.finditer(code): + if not t.group(3) and not t.group(1).startswith("-"): + hits.append(("docker", t.group(0))) + for m in RAW.finditer(code): + ref = next(g for g in m.groups() if g is not None) + if "$" not in ref and not SHA40.match(ref): + hits.append(("raw-ref", m.group(0))) + if LATEST.search(code): + hits.append(("latest", LATEST.search(code).group(0))) + for rule, what in hits: + out.append((i + 1, rule, what, why)) + return out + + +def in_scope(rel: str) -> bool: + rel = rel.replace(os.sep, "/") + if rel.startswith(".github/workflows/") and rel.endswith((".yml", ".yaml")): + return True + if rel == "CMakeLists.txt" or (rel.startswith("cmake/") and rel.endswith(".cmake")): + return True + if rel.startswith(("scripts/", "os/", "tools/")) and not rel.startswith("os/licenses/"): + return rel.endswith(".sh") or os.path.basename(rel) in ("Dockerfile", "Containerfile") + return False + + +def scan_tree(root: str) -> list[tuple[str, int, str, str, str]]: + found = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in (".git", "third_party", "build", "node_modules")] + for fn in filenames: + full = os.path.join(dirpath, fn) + rel = os.path.relpath(full, root) + if not in_scope(rel): + continue + try: + with open(full, encoding="utf-8", errors="replace") as f: + text = f.read() + except OSError: + continue + for line, rule, what, why in scan_text(rel, text): + found.append((rel, line, rule, what, why)) + return sorted(found) + + +def read_baseline(path: str) -> dict[tuple[str, str, str], str]: + rows = {} + if not os.path.exists(path): + return rows + with open(path, encoding="utf-8") as f: + for line in f: + if not line.strip() or line.startswith("#"): + continue + parts = line.rstrip("\n").split("\t") + if len(parts) >= 3: + rows[(parts[0], parts[1], parts[2])] = parts[3] if len(parts) > 3 else "" + return rows + + +def check(root: str, baseline_path: str | None = None) -> int: + found = scan_tree(root) + if baseline_path is None: + baseline_path = os.path.join(root, "tools", "ci", "moving_ref_baseline.txt") + base = read_baseline(baseline_path) + seen = set() + moving = [] + for rel, line, rule, what, why in found: + if why: + print(f"allowed {rel}:{line}: {rule}: {what} (moving-ref: {why})") + elif (rel, rule, what) in base: + seen.add((rel, rule, what)) + print(f"owed {rel}:{line}: {rule}: {what} (baseline: {base[(rel, rule, what)]})") + else: + moving.append((rel, line, rule, what)) + stale = [k for k in base if k not in seen] + for rel, line, rule, what in moving: + print(f"MOVING {rel}:{line}: {rule}: {what}") + for rel, rule, what in stale: + print(f"STALE {rel}: {rule}: {what} (baselined, no longer found: delete the row)") + if moving or stale: + print(f"moving_ref: {len(moving)} new moving input(s), {len(stale)} stale baseline row(s); pin a commit or a digest, " + "or mark a deliberate read with a `moving-ref: ` comment") + return 1 + print(f"moving_ref: every inline build input is pinned ({sum(1 for f in found if f[4])} deliberate read(s) marked, " + f"{len(seen)} baselined and owed)") + return 0 + + +def self_test() -> int: + sha = "a" * 40 + dig = "@sha256:" + "0" * 64 + cases = [ + # (path, text, expected rules) + (".github/workflows/x.yml", f" - uses: actions/checkout@{sha}\n", []), + (".github/workflows/x.yml", " - uses: actions/checkout@v4\n", ["uses"]), + (".github/workflows/x.yml", " - uses: ./.github/actions/local\n", []), + (".github/workflows/x.yml", " image: ubuntu:26.04\n", ["image"]), + (".github/workflows/x.yml", f" image: ubuntu:26.04{dig}\n", []), + (".github/workflows/x.yml", " run: python3 -m pip install numpy gguf\n", ["pip"]), + (".github/workflows/x.yml", " run: python3 -m pip install numpy==2.5.4 gguf==0.19.0\n", []), + (".github/workflows/x.yml", " run: pip install -r requirements.txt -e .\n", []), + ("os/b.sh", " -e COMPAT=x ubuntu:24.04 bash /recipe/c.sh\nexec docker run --rm ubuntu:24.04 bash\n", ["docker"]), + ("os/b.sh", f"exec docker run --rm ubuntu:24.04{dig} bash\n", []), + ("os/b.sh", 'exec docker run --rm -v "$a":/w "$image" bash\n', []), + ("os/b.sh", "curl -fsSL https://raw.githubusercontent.com/o/r/master/f.c | gcc -x c -\n", ["raw-ref"]), + ("os/b.sh", f"curl -fsSL https://raw.githubusercontent.com/o/r/{sha}/f.c | gcc -x c -\n", []), + ("os/b.sh", 'curl -fsSL "https://raw.githubusercontent.com/o/r/$ref/f.c"\n', []), + ("scripts/f.sh", "curl -L https://github.com/o/r/archive/refs/heads/main.tar.gz | tar xz\n", ["raw-ref"]), + ("scripts/f.sh", "tag=$(gh api repos/o/r/releases/latest --jq .tag_name)\n", ["latest"]), + ("scripts/f.sh", "# moving-ref: the bump bot proposes the pin\ntag=$(gh api repos/o/r/releases/latest)\n", []), + ("os/probe.sh", "curl -fsS http://127.0.0.1:8000/releases/latest\n", []), + ("os/b.sh", "# docker run ubuntu:24.04 in a comment is prose\n", []), + # the build.sh shape: the image ten continuation lines below the `docker run` + ("os/b.sh", "exec docker run --rm \\\n -v a:/b \\\n -e X=1 ubuntu:24.04 bash /r.sh\n", ["docker"]), + ("os/b.sh", f"exec docker run --rm \\\n -v a:/b \\\n -e X=1 ubuntu:24.04{dig} bash /r.sh\n", []), + ] + bad = 0 + for path, text, want in cases: + got = [r for _, r, _, why in scan_text(path, text) if not why] + ok = got == want + bad += not ok + print(f" {'PASS' if ok else 'FAIL'} {path}: {text.strip().splitlines()[-1][:70]!r} -> {got} (want {want})") + # the exemption is reported, not dropped + ex = scan_text("scripts/f.sh", "tag=$(gh api repos/o/r/releases/latest) # moving-ref: bump bot\n") + ok = len(ex) == 1 and ex[0][3] == "bump bot" + bad += not ok + print(f" {'PASS' if ok else 'FAIL'} a marked read is kept with its reason: {ex}") + # scope: a tree with one moving workflow ref and the same text out of scope + with tempfile.TemporaryDirectory() as d: + os.makedirs(os.path.join(d, ".github/workflows")) + os.makedirs(os.path.join(d, "docs")) + with open(os.path.join(d, ".github/workflows/ci.yml"), "w") as f: + f.write(" container:\n image: ubuntu:26.04\n") + with open(os.path.join(d, "docs/x.md"), "w") as f: + f.write("image: ubuntu:26.04\n") + rows = scan_tree(d) + ok = [(r[0], r[2]) for r in rows] == [(".github/workflows/ci.yml", "image")] + bad += not ok + print(f" {'PASS' if ok else 'FAIL'} scope: workflows are read, docs are not: {rows}") + # baseline: a listed finding passes, a new one fails, a stale row fails + bl = os.path.join(d, "bl.txt") + with open(bl, "w") as f: + f.write(".github/workflows/ci.yml\timage\tubuntu:26.04\tneeds workflow scope\n") + r1 = check(d, bl) + with open(bl, "a") as f: + f.write("scripts/gone.sh\tlatest\t:latest\tfixed long ago\n") + r2 = check(d, bl) + with open(bl, "w") as f: + f.write("") + r3 = check(d, bl) + ok = (r1, r2, r3) == (0, 1, 1) + bad += not ok + print(f" {'PASS' if ok else 'FAIL'} baseline: listed passes, stale row fails, unlisted fails: {(r1, r2, r3)}") + print(f"self-test: {'ok' if not bad else f'{bad} FAILED'}") + return 3 if bad else 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--check", action="store_true") + g.add_argument("--self-test", action="store_true") + ap.add_argument("--root", default=".") + ap.add_argument("--baseline", default=None) + a = ap.parse_args() + return self_test() if a.self_test else check(a.root, a.baseline) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/copyright.py b/tools/copyright.py index 219cba84f..d6ddef7ab 100755 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -67,6 +67,8 @@ "ctest_skips.txt", # engine#2018: the test-registration baseline of `tests/*_test.py`, same shape and the same reason -- data, "unreferenced_py_tests.txt", + # Gantt "Kernel 7.4 readiness": the moving-ref baseline, same shape and the same reason -- '#'-comment data. + "moving_ref_baseline.txt", "hrx_f16_narrowing.txt", # engine#2186: the accumulator baseline of `tools/ci/hrx_f16_narrowing.py --write`, same shape and the same # reason -- data, `.txt` has no comment style of its own, and the notice is already at the top of the file.