From f8ea7a212809cb8204e3157ba68b86d78a876aad Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 12:50:15 -0400 Subject: [PATCH 1/3] =?UTF-8?q?tools:=20template=5Fbom.py=20=E2=80=94=20th?= =?UTF-8?q?e=20chat=20template=20as=20a=20CycloneDX=20component=20(the=20p?= =?UTF-8?q?roposal,=20executable)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docs/template-publishing.md` proposes treating a model's `tokenizer.chat_template` as a signed part rather than an anonymous string, because a Jinja program that runs on every request should be in the dependency graph the rest of our provenance lives in. This is that proposal implemented, so the BOM, the serve log and the card row can carry the same digest instead of three bookkeepers. - `tools/template_bom.py`: for a `.gguf` (metadata header only) or a model directory (`tokenizer_config.json`'s `chat_template`, else `chat_template.jinja` — the same order the engine uses since #1095), emit a CycloneDX 1.6 `data` component with the digest in `hashes` and the provenance in `properties`; `--bom` wraps one document, default is JSON Lines so it composes. A model that carries no template gets a component with no hash and a description that says why, so "no template" is a claim a checker can verify too. - `--self-test`: two fixtures built in a temp dir (a header-only GGUF carrying a template, a directory with only `chat_template.jinja`) plus a no-template model — 7 assertions; wired into CI's python step. - `docs/template-publishing.md`: the proposal now points at the implementation, with its output. The digests it prints are the ones the merged engine prints at load: on this box `~/models/zaya1-8b-Q4_K_M.gguf` → `b55ecfbbbcbb238a…` and `~/Qwen3-0.6B-NPU2` → `a136047ba140347b…`, i.e. the same numbers `1bit serve` reports and the audit recorded, by construction rather than by bookkeeping. --- .github/workflows/ci.yml | 1 + docs/template-publishing.md | 16 ++++ tools/template_bom.py | 179 ++++++++++++++++++++++++++++++++++++ 3 files changed, 196 insertions(+) create mode 100755 tools/template_bom.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c856c631d..398302ec6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,7 @@ jobs: python3 tests/mtp_draft_vocab_test.py python3 tests/check_pins_test.py python3 tests/hrx_long_prompt_test.py + python3 tools/template_bom.py --self-test # the chat-template BOM component - name: Configure run: cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=gcc-16 -DCMAKE_CXX_COMPILER=g++-16 - name: Build diff --git a/docs/template-publishing.md b/docs/template-publishing.md index 90c48210d..889e86adb 100644 --- a/docs/template-publishing.md +++ b/docs/template-publishing.md @@ -137,3 +137,19 @@ instead of only checking a pin. No upstream filing, no vendor contact: this is a proposal for our own BOM output (`tools/` and the release packages) and for the cards we publish. + +**`tools/template_bom.py` emits exactly that component** for a `.gguf` or a model directory — the digest +in `hashes`, the file it came from in `properties`, `--bom` for a whole CycloneDX 1.6 document, and a +component with **no** hash and a description for a model that carries no template (so "no template" is a +verifiable claim in the BOM too): + +```console +$ tools/template_bom.py --bom ~/models/zaya1-8b-Q4_K_M.gguf +{"bomFormat": "CycloneDX", "specVersion": "1.6", "version": 1, "components": [ + {"type": "data", "bom-ref": "pkg:generic/zaya1-8b-Q4_K_M.gguf?chat-template", + "name": "tokenizer.chat_template", + "hashes": [{"alg": "SHA-256", "content": "b55ecfbbbcbb238a011465bfa139366db18790103f9205060cf2855a3f51b30f"}], …}]} +``` + +That digest is the same one `1bit serve` prints at load, so the BOM, the log and the card row agree by +construction rather than by bookkeeping. diff --git a/tools/template_bom.py b/tools/template_bom.py new file mode 100755 index 000000000..0c494f6be --- /dev/null +++ b/tools/template_bom.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The chat template as a CycloneDX component, so a BOM can sign and diff it. + +A `tokenizer.chat_template` is a Jinja program the llama.cpp-family backends run on every request, and +it has no name, no digest and no place in a dependency graph — which is why it escapes review. Give it +the shape of a component and the tools that already read a BOM start covering it: the digest goes in +`hashes`, where it came from goes in `properties`, and "the weights are the same file and the template +is not" becomes a component difference instead of an invisible substitution +(docs/template-publishing.md, "Proposal: the chat template as a signed part"). + +usage: tools/template_bom.py [--bom] [--self-test] PATH... + + PATH a .gguf file, or a model directory (Q4NX/NPU container) — the template is read from the GGUF's + metadata header, or from the directory's tokenizer_config.json `chat_template`, else + `chat_template.jinja` beside it. Only the header is read; the tensors never are. + --bom emit one CycloneDX 1.6 document with a component per PATH; without it, one component per line + (JSON Lines), so it composes with the other tools. + --self-test build two tiny fixtures (a GGUF with a template, a directory with chat_template.jinja) and + check the components this tool derives from them. + +Exit codes: 0 all paths read, 1 a path could not be read, 3 --self-test failed. +""" +from __future__ import annotations + +import hashlib +import json +import os +import sys +import tempfile + +SPEC_VERSION = "1.6" + + +def read_template(path: str) -> tuple[str | None, str | None]: + """(template text, where it came from) — `(None, None)` when the model carries none.""" + if os.path.isdir(path): + cfg = os.path.join(path, "tokenizer_config.json") + if os.path.exists(cfg): + try: + with open(cfg, encoding="utf-8") as f: + t = json.load(f).get("chat_template") + if isinstance(t, str) and t: + return t, "tokenizer_config.json" + except Exception: + pass + jinja = os.path.join(path, "chat_template.jinja") + if os.path.exists(jinja): + with open(jinja, encoding="utf-8") as f: + return f.read(), "chat_template.jinja" + return None, None + import gguf # only for a .gguf: keeps --self-test's directory case dependency-free + + reader = gguf.GGUFReader(path) + field = reader.fields.get("tokenizer.chat_template") + if field is None: + return None, None + return bytes(field.parts[-1]).decode("utf-8", "replace"), "GGUF metadata (tokenizer.chat_template)" + + +def component(path: str) -> dict: + tpl, source = read_template(path) + ref = "pkg:generic/" + os.path.basename(os.path.abspath(path.rstrip("/"))) + "?chat-template" + props = [ + {"name": "onebit:template:path", "value": os.path.abspath(path)}, + {"name": "onebit:template:source-file", "value": source or "none"}, + ] + if tpl is None: + return { + "type": "data", + "bom-ref": ref, + "name": "tokenizer.chat_template", + "description": "no chat template in this model (a base model, an mmproj, or a container without one)", + "properties": props, + } + props.insert(1, {"name": "onebit:template:bytes", "value": str(len(tpl.encode()))}) + return { + "type": "data", + "bom-ref": ref, + "name": "tokenizer.chat_template", + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(tpl.encode()).hexdigest()}], + "properties": props, + } + + +def self_test() -> int: + import struct + + failures = 0 + + def expect(what: str, got, want) -> None: + ok = got == want + print(f"{'ok ' if ok else 'FAIL'} {what}: {got!r}") + nonlocal failures + failures += 0 if ok else 1 + + tpl = "{% for m in messages %}{{ '' ~ m['role'] }}{% endfor %}assistant\\n" + with tempfile.TemporaryDirectory() as d: + # a header-only GGUF carrying the template, written the way a real one stores it + def s(x: str) -> bytes: + b = x.encode() + return struct.pack(" int: + if "--self-test" in argv: + return self_test() + as_bom = "--bom" in argv + paths = [a for a in argv if not a.startswith("-")] + if not paths: + print(__doc__.strip().splitlines()[0], file=sys.stderr) + print("usage: tools/template_bom.py [--bom] [--self-test] PATH...", file=sys.stderr) + return 2 + comps, failed = [], False + for p in paths: + try: + comps.append(component(p)) + except Exception as e: + print(f"template_bom: {p}: {e}", file=sys.stderr) + failed = True + if as_bom: + print(json.dumps({"bomFormat": "CycloneDX", "specVersion": SPEC_VERSION, "version": 1, + "components": comps}, indent=1)) + else: + for c in comps: + print(json.dumps(c, sort_keys=True)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From bd48b3dc0cf2042d16cf1ee4cc846eade9cbcc7d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 13:20:12 -0400 Subject: [PATCH 2/3] tools/template_bom.py: --sums, the SHA256SUMS line a published file needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool emitted the CycloneDX component, and the publishing rule's other half — the `SHA256SUMS` line (` #tokenizer.chat_template`, or `/#chat_template` for a container) — was still the snippet in the doc. Now one command gives both: tools/template_bom.py --sums model.gguf container-dir/ b55ecfbbbcbb238a… zaya1-8b-Q4_K_M.gguf#tokenizer.chat_template a136047ba140347b… /path/Qwen3-0.6B-NPU2/tokenizer_config.json#chat_template A model that carries no template prints nothing, so the file a publisher appends to stays clean rather than gaining a 'none' row. Self-test grows to 10 assertions (the two keys, and the no-template silence). --- tools/template_bom.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tools/template_bom.py b/tools/template_bom.py index 0c494f6be..200e71510 100755 --- a/tools/template_bom.py +++ b/tools/template_bom.py @@ -22,13 +22,16 @@ is not" becomes a component difference instead of an invisible substitution (docs/template-publishing.md, "Proposal: the chat template as a signed part"). -usage: tools/template_bom.py [--bom] [--self-test] PATH... +usage: tools/template_bom.py [--bom | --sums] [--self-test] PATH... PATH a .gguf file, or a model directory (Q4NX/NPU container) — the template is read from the GGUF's metadata header, or from the directory's tokenizer_config.json `chat_template`, else `chat_template.jinja` beside it. Only the header is read; the tensors never are. --bom emit one CycloneDX 1.6 document with a component per PATH; without it, one component per line (JSON Lines), so it composes with the other tools. + --sums emit the `SHA256SUMS` lines instead — ` #tokenizer.chat_template` for a `.gguf`, + `/#chat_template` for a container — the row a published file needs + (docs/template-publishing.md). 'none' models print nothing, so the file stays clean. --self-test build two tiny fixtures (a GGUF with a template, a directory with chat_template.jinja) and check the components this tool derives from them. @@ -96,6 +99,17 @@ def component(path: str) -> dict: } +def sums_line(path: str) -> str | None: + """The `SHA256SUMS` line for PATH, or None when the model carries no template.""" + tpl, source = read_template(path) + if tpl is None: + return None + sha = hashlib.sha256(tpl.encode()).hexdigest() + if os.path.isdir(path): + return f"{sha} {os.path.join(path, os.path.basename(source)) if source else path}#chat_template" + return f"{sha} {os.path.basename(path)}#tokenizer.chat_template" + + def self_test() -> int: import struct @@ -147,6 +161,9 @@ def s(x: str) -> bytes: expect("no template: no hashes", "hashes" in c, False) expect("no template: source", next(p["value"] for p in c["properties"] if p["name"] == "onebit:template:source-file"), "none") + expect("gguf: sums key", sums_line(gguf_path).split(" ")[1], "tpl.gguf#tokenizer.chat_template") + expect("dir: sums key", sums_line(cont).split(" ")[1].endswith("cont/chat_template.jinja#chat_template"), True) + expect("no template: no sums line", sums_line(bare), None) return 3 if failures else 0 @@ -154,10 +171,11 @@ def main(argv: list[str]) -> int: if "--self-test" in argv: return self_test() as_bom = "--bom" in argv + as_sums = "--sums" in argv paths = [a for a in argv if not a.startswith("-")] if not paths: print(__doc__.strip().splitlines()[0], file=sys.stderr) - print("usage: tools/template_bom.py [--bom] [--self-test] PATH...", file=sys.stderr) + print("usage: tools/template_bom.py [--bom | --sums] [--self-test] PATH...", file=sys.stderr) return 2 comps, failed = [], False for p in paths: @@ -166,7 +184,15 @@ def main(argv: list[str]) -> int: except Exception as e: print(f"template_bom: {p}: {e}", file=sys.stderr) failed = True - if as_bom: + if as_sums: + for p in paths: + try: + line = sums_line(p) + except Exception: + line = None + if line: + print(line) + elif as_bom: print(json.dumps({"bomFormat": "CycloneDX", "specVersion": SPEC_VERSION, "version": 1, "components": comps}, indent=1)) else: From d82ae927df17dadc4030684155d14cd3646854ef Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 13:58:16 -0400 Subject: [PATCH 3/3] tools/template_bom.py: digest the template bytes on disk, so the BOM and the serve log agree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review: on a CRLF container the component's digest and the digest `1bit serve` prints were two different sha256 values for one template, because the directory branch read chat_template.jinja in text mode — Python's universal-newline translation — while `container_chat_template_sha()` reads the file as binary and `docs/template-publishing.md` tells a reader to check it with `sha256sum`. $ sha256sum cont/chat_template.jinja d66cf707b336fdfd… $ template_bom.py --sums cont/ 580f68ef79d2ecf9… (before: 1314 B, LF-normalised) d66cf707b336fdfd… (after: the file's own digest) read_template() now returns bytes and every branch hashes what is stored: the .jinja file as `"rb"`, the GGUF metadata part as-is, and tokenizer_config.json's JSON string encoded (a JSON string carries no raw newline, so it already equalled the C++ json read). --self-test gains a CRLF fixture written with `"wb"` whose expected digest is sha256 of the bytes written, plus an assertion that it is NOT the LF-normalised digest — that pair fails on the previous head and 14/14 pass here. The two recorded digests are unchanged, so no committed pin or card row moves: zaya1-8b-Q4_K_M.gguf b55ecfbbbcbb238a…, ~/Qwen3-0.6B-NPU2 a136047ba140347b…. --- docs/template-publishing.md | 4 +++- tools/template_bom.py | 42 ++++++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/template-publishing.md b/docs/template-publishing.md index 889e86adb..48ecd83ca 100644 --- a/docs/template-publishing.md +++ b/docs/template-publishing.md @@ -152,4 +152,6 @@ $ tools/template_bom.py --bom ~/models/zaya1-8b-Q4_K_M.gguf ``` That digest is the same one `1bit serve` prints at load, so the BOM, the log and the card row agree by -construction rather than by bookkeeping. +construction rather than by bookkeeping. Both branches hash the bytes as stored — a container whose +`chat_template.jinja` is CRLF gets its `sha256sum`, not a newline-normalised variant — because that is +what the serve path hashes too. diff --git a/tools/template_bom.py b/tools/template_bom.py index 200e71510..f68a662fd 100755 --- a/tools/template_bom.py +++ b/tools/template_bom.py @@ -32,8 +32,8 @@ --sums emit the `SHA256SUMS` lines instead — ` #tokenizer.chat_template` for a `.gguf`, `/#chat_template` for a container — the row a published file needs (docs/template-publishing.md). 'none' models print nothing, so the file stays clean. - --self-test build two tiny fixtures (a GGUF with a template, a directory with chat_template.jinja) and - check the components this tool derives from them. + --self-test build three tiny fixtures (a GGUF with a template, a container directory whose template is + LF and CRLF, and a model that carries none) and check the components this tool derives from them. Exit codes: 0 all paths read, 1 a path could not be read, 3 --self-test failed. """ @@ -48,8 +48,13 @@ SPEC_VERSION = "1.6" -def read_template(path: str) -> tuple[str | None, str | None]: - """(template text, where it came from) — `(None, None)` when the model carries none.""" +def read_template(path: str) -> tuple[bytes | None, str | None]: + """(the template bytes as stored, where they came from) — `(None, None)` when the model carries none. + + Bytes, never decoded text: the served side (`container_chat_template_sha()`, `app/template_pin.h`) + hashes the raw file and so does `sha256sum`, so a text read's newline translation would name a + digest the file does not have — on a CRLF container that is a different sha256 for one template. + """ if os.path.isdir(path): cfg = os.path.join(path, "tokenizer_config.json") if os.path.exists(cfg): @@ -57,12 +62,13 @@ def read_template(path: str) -> tuple[str | None, str | None]: with open(cfg, encoding="utf-8") as f: t = json.load(f).get("chat_template") if isinstance(t, str) and t: - return t, "tokenizer_config.json" + # a JSON string value carries no raw newline, so this equals the C++ json read + return t.encode(), "tokenizer_config.json" except Exception: pass jinja = os.path.join(path, "chat_template.jinja") if os.path.exists(jinja): - with open(jinja, encoding="utf-8") as f: + with open(jinja, "rb") as f: return f.read(), "chat_template.jinja" return None, None import gguf # only for a .gguf: keeps --self-test's directory case dependency-free @@ -71,7 +77,7 @@ def read_template(path: str) -> tuple[str | None, str | None]: field = reader.fields.get("tokenizer.chat_template") if field is None: return None, None - return bytes(field.parts[-1]).decode("utf-8", "replace"), "GGUF metadata (tokenizer.chat_template)" + return bytes(field.parts[-1]), "GGUF metadata (tokenizer.chat_template)" def component(path: str) -> dict: @@ -89,12 +95,12 @@ def component(path: str) -> dict: "description": "no chat template in this model (a base model, an mmproj, or a container without one)", "properties": props, } - props.insert(1, {"name": "onebit:template:bytes", "value": str(len(tpl.encode()))}) + props.insert(1, {"name": "onebit:template:bytes", "value": str(len(tpl))}) return { "type": "data", "bom-ref": ref, "name": "tokenizer.chat_template", - "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(tpl.encode()).hexdigest()}], + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(tpl).hexdigest()}], "properties": props, } @@ -104,7 +110,7 @@ def sums_line(path: str) -> str | None: tpl, source = read_template(path) if tpl is None: return None - sha = hashlib.sha256(tpl.encode()).hexdigest() + sha = hashlib.sha256(tpl).hexdigest() if os.path.isdir(path): return f"{sha} {os.path.join(path, os.path.basename(source)) if source else path}#chat_template" return f"{sha} {os.path.basename(path)}#tokenizer.chat_template" @@ -152,6 +158,22 @@ def s(x: str) -> bytes: expect("dir: source", next(p["value"] for p in c["properties"] if p["name"] == "onebit:template:source-file"), "chat_template.jinja") + # the same template with CRLF line endings, written as bytes: the digest must be the file's, not + # Python's universal-newline view of it (app/template_pin.h hashes the raw bytes too) + crlf_tpl = "{% for m in messages %}\r\n{{ '' ~ m['role'] }}\r\n{% endfor %}\r\n" + raw = crlf_tpl.encode() + crlf = os.path.join(d, "crlf") + os.makedirs(crlf) + with open(os.path.join(crlf, "chat_template.jinja"), "wb") as f: + f.write(raw) + c = component(crlf) + expect("crlf: sha256 is the file's", c["hashes"][0]["content"], hashlib.sha256(raw).hexdigest()) + expect("crlf: not the LF-normalised digest", + c["hashes"][0]["content"] != hashlib.sha256(raw.replace(b"\r\n", b"\n")).hexdigest(), True) + expect("crlf: bytes", next(p["value"] for p in c["properties"] if p["name"] == "onebit:template:bytes"), + str(len(raw))) + expect("crlf: sums digest", sums_line(crlf).split(" ")[0], hashlib.sha256(raw).hexdigest()) + # a model that carries no template: a component with no hash and a description that says why bare = os.path.join(d, "bare") os.makedirs(bare)