From a26ef180bfddda3cfe58c2f668918b309fe6d7e1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 14:21:49 -0400 Subject: [PATCH 1/9] tools: mtp_depth_pool.py -- the pooled per-depth live-vs-offline verdict, with its traps as refusals The MTP brief's clause is judged on 8 prompts x 64 tokens POOLED, but one `mtp_acceptance --live` run costs ~90 s of array time plus a model + head load, so an 8-prompt window is two or three npu-hold tickets of <= 3 min. Pooling those logs by eye produced two wrong numbers in this lane, so the arithmetic moves into the tree and the two traps become refusals: * a ticket log REPEATS ITS TAIL after the harness' exit line (pooling that inflated a 12-cell window to 20) -- everything after '== ticket exited' is ignored; * a cell is keyed by (prompt, depth); a second row for a key with different counts is a disagreement between tickets or a format change, never an average; * a ticket's own run-level row must equal the sum of its per-prompt rows, so a truncated log cannot pass as a smaller window; * a depth with no denominator is UNMEASURED, not a +0.00 pp pass (the phantom-depth defect, #916). Known-answer check, on the real logs that produced the recorded numbers: pointing the tool at the three 2026-11-05 clause tickets reproduces 136/181, 86/136, 63/86, 53/63 and the recorded deltas (+0.00 / +0.55 / +0.09 / +1.08 pp) to the count, so this port is equivalent to the script that made the claim. tests/mtp_depth_pool_test.py drives fixture tickets (no model, no device) through the agreeing, duplicated, disagreeing, short-read and empty-denominator cases; registered as ctest mtp_depth_pool. --- CMakeLists.txt | 5 + tests/mtp_depth_pool_test.py | 141 ++++++++++++++++++++++ tools/mtp_depth_pool.py | 227 +++++++++++++++++++++++++++++++++++ 3 files changed, 373 insertions(+) create mode 100755 tests/mtp_depth_pool_test.py create mode 100755 tools/mtp_depth_pool.py diff --git a/CMakeLists.txt b/CMakeLists.txt index effeaf947..6a2b2d55f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -566,6 +566,11 @@ if(Python3_Interpreter_FOUND) # WIDER tile grid than the model's dimensions (the FastFlowLM repack pads intermediate_size to 512 columns, IM # 8960 -> 9216) instead of reporting it as an unreadable reshape. Pure python + numpy, no model, no device. add_test(NAME q4nx_diff_grid COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/q4nx_diff_grid_test.py) + # The pooled per-depth live-vs-offline verdict (tools/mtp_depth_pool.py): the brief's "within 2 pp at every + # depth" is judged on 8 prompts x 64 tokens POOLED, which is two or three npu-hold tickets of <= 3 min, and + # pooling those logs by eye produced two wrong numbers in this lane. Fixture tickets only -- no model, no + # device; the recorded 8x64 window is the tool's known-answer check and is run by hand with --selftest. + add_test(NAME mtp_depth_pool COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/mtp_depth_pool_test.py) # the t2 chunk kind (docs/npu.md "The t2 chunk kind"): synthetic only, because the real-model # leg needs 14 GB of Bonsai-2 images. Run tests/q4nx_t2_test.py without --no-model for the gate. add_test(NAME q4nx_t2 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/q4nx_t2_test.py --no-model) diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py new file mode 100755 index 000000000..928cace95 --- /dev/null +++ b/tests/mtp_depth_pool_test.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tools/mtp_depth_pool.py: the pooled per-depth live-vs-offline verdict, against fixture tickets. + +The tool exists because the brief judges "live acceptance equals the offline protocol within 2 pp at every +depth" on 8 prompts x 64 tokens POOLED, while a device ticket can only carry two or three prompts, and because +pooling `mtp_acceptance --live --match` logs by eye produced two wrong numbers in this lane. Both traps are +refusals in the tool, so they are tested here with fake tickets instead of only on the happy path: + +1. two tickets that agree pool; a ticket whose per-prompt rows disagree on a shared cell, or that does not add + up to its own run-level row (a short read), is refused rather than averaged; +2. the echoed summary a harness prints after its exit line is ignored, so a cell is never counted twice; +3. a depth with no denominator on either side is reported as unmeasured, never as a +0.00 pp pass. + +usage: tests/mtp_depth_pool_test.py (pure python, no model, no device) +""" +import os +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TOOL = os.path.join(ROOT, "tools", "mtp_depth_pool.py") + +# prompt 0: depth1 8/10 (offline 8/10), depth2 5/9 (5/9) +# prompt 1: depth1 7/10 (offline 6/10), depth2 4/9 (4/9) +TICKET_A = """forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +prompt 0: TEXT IDENTICAL, 64 tokens, accepted 6/16, passes/token 0.500, stream 72 rows, 0 mismatch + match p0: 4 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10) +depth 2: live 55.56 % (5/9, reached) offline conditional 55.56 % (5/9) +prompt 1: TEXT IDENTICAL, 64 tokens, accepted 5/16, passes/token 0.500, stream 72 rows, 0 mismatch + match p1: 4 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 70.00 % (7/10, reached) offline conditional 60.00 % (6/10) +depth 2: live 44.44 % (4/9, reached) offline conditional 44.44 % (4/9) +depth 1: live 75.00 % (15/20) offline 70.00 % (14/20) delta +5.00 pp FAIL +depth 2: live 50.00 % (9/18) offline 50.00 % (9/18) delta +0.00 pp PASS +== ticket exited rc=0 (hold reached) +prompt 0: TEXT IDENTICAL, 64 tokens, accepted 6/16, passes/token 0.500, stream 72 rows, 0 mismatch +depth 1: live 40.00 % (4/10, reached) offline conditional 40.00 % (4/10) +""" + +# prompt 2: depth1 9/10 (offline 7/10), depth2 3/9 (3/9) +TICKET_B = """prompt 2: TEXT IDENTICAL, 64 tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch + match p2: 4 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 90.00 % (9/10, reached) offline conditional 70.00 % (7/10) +depth 2: live 33.33 % (3/9, reached) offline conditional 33.33 % (3/9) +depth 1: live 90.00 % (9/10) offline 70.00 % (7/10) delta +20.00 pp FAIL +depth 2: live 33.33 % (3/9) offline 33.33 % (3/9) delta +0.00 pp PASS +""" + +# pooled over A+B: depth1 live 24/30 = 80.00 % vs offline 21/30 = 70.00 % -> +10.00 pp FAIL +# depth2 live 12/27 = 44.44 % vs offline 12/27 = 44.44 % -> +0.00 pp PASS +POOLED_DEPTH1 = "24/30 = 80.00%" +POOLED_DEPTH2 = "12/27 = 44.44%" + +fails = 0 + + +def check(cond, what): + global fails + print(f"{'PASS' if cond else 'FAIL'} {what}") + if not cond: + fails += 1 + + +def run(args, logs): + paths = [] + for name, body in logs: + p = os.path.join(TMP, name) + with open(p, "w") as fh: + fh.write(body) + paths.append(p) + r = subprocess.run([sys.executable, TOOL] + args + paths, capture_output=True, text=True) + return r.returncode, r.stdout, r.stderr + + +TMP = tempfile.mkdtemp(prefix="mtp-depth-pool-test.") + +# 1. two tickets that agree: the pool is their union, keyed by (prompt, depth), and the tail echo is ignored. +rc, out, err = run([], [("a.log", TICKET_A), ("b.log", TICKET_B)]) +check(rc == 0, "two agreeing tickets pool without a refusal") +check("pooled 6 scored cells = 3 prompts x 2 depths" in out, "3 prompts x 2 depths = 6 cells (the echoed tail is not counted)") +check(POOLED_DEPTH1 in out, f"depth 1 pools to {POOLED_DEPTH1}") +check(POOLED_DEPTH2 in out, f"depth 2 pools to {POOLED_DEPTH2}") +check("+10.00 pp" in out and "FAIL" in out, "depth 1 is 10.00 pp out, so the depth itself reads FAIL") +check("1 of 2 scored depths within 2 pp" in out, "the verdict line counts one pass of two scored depths") + +# 2. the same ticket twice is not double counting: identical cells are the same cells. +rc2, out2, _ = run([], [("a1.log", TICKET_A), ("a2.log", TICKET_A)]) +check(rc2 == 0 and "pooled 4 scored cells" in out2, "the same ticket twice pools to the same 4 cells, not 8") + +# 3. two tickets disagreeing on one shared cell -- prompt 1 depth 2, which both A and C score -- is refused. +# C is internally consistent (its own run-level row matches its rows), so only the cross-ticket check can catch it. +TICKET_C = """prompt 1: TEXT IDENTICAL, 64 tokens, accepted 5/16, passes/token 0.500, stream 72 rows, 0 mismatch +depth 1: live 70.00 % (7/10, reached) offline conditional 60.00 % (6/10) +depth 2: live 44.44 % (4/9, reached) offline conditional 11.11 % (1/9) +depth 1: live 70.00 % (7/10) offline 60.00 % (6/10) delta +10.00 pp FAIL +depth 2: live 44.44 % (4/9) offline 11.11 % (1/9) delta +33.33 pp FAIL +""" +rc3, _, err3 = run([], [("a.log", TICKET_A), ("c.log", TICKET_C)]) +check(rc3 == 1, f"a disagreement between tickets is refused (rc={rc3})") +check("disagrees" in err3 or "scored twice" in err3, f"the refusal says what disagrees: {err3.strip()[:120]}") + +# 4. a short read: the per-prompt rows no longer add up to the ticket's own run-level row. +short = TICKET_A.replace("depth 2: live 44.44 % (4/9, reached) offline conditional 44.44 % (4/9)\n", "") +rc4, _, err4 = run([], [("short.log", short)]) +check(rc4 == 1, f"a log missing a cell is refused (rc={rc4})") +check("read incompletely" in err4, f"the refusal names the short read: {err4.strip()[:120]}") + +# 5. a depth with no denominator is unmeasured, not a pass. +floor0 = """prompt 0: TEXT IDENTICAL, 64 tokens, accepted 1/16, passes/token 0.500, stream 72 rows, 0 mismatch +depth 1: live 100.00 % (1/1, reached) offline conditional 100.00 % (1/1) +depth 2: live 0.00 % (0/0, reached) offline conditional 0.00 % (0/0) +depth 1: live 100.00 % (1/1) offline 100.00 % (1/1) delta +0.00 pp PASS +depth 2: live 0.00 % (0/0) offline 0.00 % (0/0) delta +0.00 pp PASS +""" +rc5, out5, _ = run([], [("floor0.log", floor0)]) +check(rc5 == 0, "a depth with an empty denominator does not fail the run") +check("no denominator" in out5 and "1 of 1 scored depths" in out5, + "the empty depth is reported as unmeasured and excluded from the scored count") + +# 6. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) +check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") + +print(f"\n{'all checks passed' if not fails else f'{fails} CHECK(S) FAILED'}") +sys.exit(1 if fails else 0) diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py new file mode 100755 index 000000000..01e6f7bf5 --- /dev/null +++ b/tools/mtp_depth_pool.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# mtp_depth_pool.py -- pool the live-vs-offline per-depth table over several `mtp_acceptance --live --match` +# runs, and say whether the brief's clause ("live acceptance equals the offline protocol within 2 pp at every +# depth") holds on the pooled window. +# +# Why pooling exists at all: the clause is judged on 8 prompts x 64 tokens POOLED, but one `--live` run costs +# ~90 s of array time plus a model + head load per process, so an 8-prompt window is usually two or three +# npu-hold tickets of <= 3 min each. Pooling those logs by eye is how this lane produced two wrong numbers, +# so the arithmetic lives here and the two traps are refusals rather than comments: +# 1. a ticket log REPEATS ITS OWN TAIL after the harness' exit line, so a cell can appear twice -- pooling +# that once inflated a 12-cell window to 20 cells; +# 2. a cell is keyed by (prompt, depth), and a second row for the same key with different counts is a +# disagreement between two tickets (or a format change), never an average to be taken. +# A ticket's own run-level row is the sum of its per-prompt rows, so it is checked too: a truncated log cannot +# then pass silently as a smaller window. +# +# tools/mtp_depth_pool.py ticket-a.log ticket-b.log ... # pooled table + the 2 pp verdict per depth +# tools/mtp_depth_pool.py --selftest # the refusals and the arithmetic, no logs needed +# tools/mtp_depth_pool.py --selftest # ... and the recorded 8x64 window, to the count +# +# Exit: 0 the pool was computed (individual depths may still read FAIL), 1 a refusal, 2 no input. +"""tools/mtp_depth_pool.py -- the per-depth live-vs-offline verdict over several mtp_acceptance tickets.""" +import re +import sys + +BAR = 2.0 # the brief's bar, in percentage points + +# The tool prints the per-depth table in two shapes, and confusing them is what this file exists not to repeat: +# per PROMPT (keyed, poolable): +# depth 2: live 55.00 % (11/20, reached) offline conditional 52.63 % (10/19) delta +2.37 pp > 2 pp +# per RUN (this ticket's own pool over its prompts, used only as the read-completeness check): +# depth 2: live 52.78 % (38/72) offline 52.11 % (37/71) delta +0.67 pp PASS +DETAIL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+), reached\)\s+" + r"offline conditional\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)") +RUNLEVEL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+" + r"offline\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+delta") +PROMPT = re.compile(r"(?:per-depth p|prompt )(\d+)\b") + +# The numbers this tool must reproduce when it is pointed at the recorded window (TMTPLOOP-STATE.md, +# 2026-11-05): live (num, den) and offline (num, den) per depth, over 8 prompts x 64 tokens. +RECORDED = {1: (136, 181, 136, 181), 2: (86, 136, 84, 134), 3: (63, 86, 60, 82), 4: (53, 63, 49, 59)} + + +def parse_ticket(path): + """Return {prompt, depth: (live_n, live_d, off_n, off_d)} for one ticket log. + + Raises SystemExit on a log it must not pool: a depth row before any prompt header (nothing to key it to), + a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own + run-level row (a short read).""" + cells, runlevel, prompt = {}, {}, None + for ln in open(path, errors="replace").read().splitlines(): + if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary + break + m = PROMPT.search(ln) + if m and "depth " not in ln: + prompt = int(m.group(1)) + continue + m = DETAIL.search(ln) + if m: + if prompt is None: + raise SystemExit(f"mtp_depth_pool: {path}: a depth row before any prompt line -- " + f"refusing to guess which prompt it belongs to") + key = (prompt, int(m.group(1))) + got = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) + if key in cells and cells[key] != got: + raise SystemExit(f"mtp_depth_pool: {path}: prompt {key[0]} depth {key[1]} scored twice with " + f"different counts {cells[key]} then {got}") + cells.setdefault(key, got) + continue + m = RUNLEVEL.search(ln) + if m: + runlevel[int(m.group(1))] = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) + # The ticket's own run-level row is the pool of its per-prompt rows, so a short read cannot pass as a + # smaller window: the two have to agree before this log is allowed into the pool at all. + for d, rl in runlevel.items(): + got = tuple(sum(v[i] for (_, dd), v in cells.items() if dd == d) for i in range(4)) + if got != rl: + raise SystemExit(f"mtp_depth_pool: {path}: depth {d} per-prompt rows pool to {got} but the ticket's " + f"own run-level row says {rl} -- the log was read incompletely, do not pool it") + if runlevel and len(runlevel) != len({d for _, d in cells}): + raise SystemExit(f"mtp_depth_pool: {path}: run-level row covers {sorted(runlevel)} but per-prompt rows " + f"cover {sorted({d for _, d in cells})}") + return cells + + +def pool(paths): + """Union the tickets' cells, refusing any disagreement on a key. Returns (cells, sources).""" + seen, sources = {}, 0 + for p in paths: + rows = parse_ticket(p) + if not rows: + print(f"mtp_depth_pool: {p}: no matched rows (was it run with --live --match?)", file=sys.stderr) + continue + sources += 1 + for k, v in rows.items(): + if k in seen and seen[k] != v: + raise SystemExit(f"mtp_depth_pool: {p} disagrees with an earlier ticket on prompt {k[0]} " + f"depth {k[1]}: {seen[k]} then {v}") + seen.setdefault(k, v) + return seen, sources + + +def table(seen, sources): + """Print the pooled per-depth table; return (inside, scored, totals per depth).""" + depths = sorted({d for _, d in seen}) + prompts = sorted({p for p, _ in seen}) + print(f"pooled {len(seen)} scored cells = {len(prompts)} prompts x {len(depths)} depths " + f"(prompts {prompts[0]}..{prompts[-1]}, {sources} ticket(s))") + print(f"\n{'depth':>5} {'live':>18} {'offline (same positions)':>24} {'delta':>9} verdict") + inside = scored = 0 + totals = {} + for d in depths: + live_n = sum(v[0] for (_, dd), v in seen.items() if dd == d) + live_d = sum(v[1] for (_, dd), v in seen.items() if dd == d) + off_n = sum(v[2] for (_, dd), v in seen.items() if dd == d) + off_d = sum(v[3] for (_, dd), v in seen.items() if dd == d) + totals[d] = (live_n, live_d, off_n, off_d) + if not live_d or not off_d: + # 0/0 is not agreement: a depth no chain reached is unmeasured, and scoring it as +0.00 pp PASS + # is how a phantom depth was once reported as a passing one (engine#916). + print(f"{d:>5} {'no denominator':>18}") + continue + delta = 100.0 * live_n / live_d - 100.0 * off_n / off_d + ok = abs(delta) <= BAR + scored += 1 + inside += ok + print(f"{d:>5} {f'{live_n}/{live_d} = {100.0 * live_n / live_d:.2f}%':>18} " + f"{f'{off_n}/{off_d} = {100.0 * off_n / off_d:.2f}%':>24} {f'{delta:+.2f} pp':>9} " + f"{'PASS' if ok else 'FAIL'}") + print(f"\n{inside} of {scored} scored depths within {BAR:g} pp at the same positions") + return inside, scored, totals + + +def refusal_selftest(): + """The refusals are why this tool exists, so they are tested and not only the happy path.""" + import os + import tempfile + + d2 = "depth 2: live 72.22 % (13/18, reached) offline conditional 72.22 % (13/18)\n" + cases = [ + # a cell scored twice with different counts (two tickets disagreeing, or a format change) + ("differing duplicate", "prompt 0: TEXT IDENTICAL\n" + d2 + + "depth 2: live 50.00 % (9/18, reached) offline conditional 50.00 % (9/18)\n", "scored twice"), + # a log whose per-prompt rows do not add up to the ticket's own run-level row (short read) + ("incomplete read", "prompt 0: TEXT IDENTICAL\n" + d2 + + "depth 2: live 50.00 % (9/18) offline 50.00 % (9/18) delta +0.00 pp PASS\n", + "read incompletely"), + # a depth row with no prompt header to key it to + ("unkeyed depth row", d2, "before any prompt line"), + ] + for name, body, want in cases: + with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as fh: + fh.write(body) + path = fh.name + try: + parse_ticket(path) + except SystemExit as e: + if want not in str(e): + print(f"selftest FAIL {name}: refused with {str(e)!r}, wanted {want!r}", file=sys.stderr) + return 1 + print(f"selftest PASS {name}: refused -- {str(e).split(': ', 1)[-1][:96]}") + else: + print(f"selftest FAIL {name}: accepted a log it must refuse", file=sys.stderr) + return 1 + finally: + os.unlink(path) + + # The tail trap: the same cells repeated after the exit line must not double the window. + d1 = "depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10)\n" + run = "depth 1: live 80.00 % (8/10) offline 80.00 % (8/10) delta +0.00 pp PASS\n" + tail = ("== ticket exited rc=0 (hold reached)\n" + "prompt 0: TEXT IDENTICAL\n" + d1 + + "depth 1: live 40.00 % (4/10, reached) offline conditional 40.00 % (4/10)\n") + with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as fh: + fh.write("prompt 0: TEXT IDENTICAL\n" + d1 + run + tail) + path = fh.name + try: + cells = parse_ticket(path) + if cells != {(0, 1): (8, 10, 8, 10)}: + print(f"selftest FAIL tail repeat: parsed {cells}, wanted only the first row", file=sys.stderr) + return 1 + print("selftest PASS tail repeat: the echoed summary after the exit line is ignored") + finally: + os.unlink(path) + return 0 + + +def main(argv): + selftest = "--selftest" in argv + paths = [a for a in argv if a != "--selftest"] + if not paths: + if selftest: + return refusal_selftest() + print(__doc__) + return 2 + seen, sources = pool(paths) + if not seen: + print("mtp_depth_pool: nothing to pool", file=sys.stderr) + return 1 + _, _, totals = table(seen, sources) + if not selftest: + return 0 + bad = {d: (totals.get(d), RECORDED[d]) for d in RECORDED if totals.get(d) != RECORDED[d]} + if bad: + for d, (g, e) in bad.items(): + print(f"selftest FAIL depth {d}: pooled {g} != recorded {e}", file=sys.stderr) + return 1 + print(f"selftest PASS: {len(RECORDED)} depths reproduce the recorded 8x64 window to the count") + return refusal_selftest() + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 9ec6cce8d7ba635fe427009f507bc8422cdee3d6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:13:53 -0400 Subject: [PATCH 2/9] tools: the pooled verdict names the depths it could not score and the prompts it did not cover mtp_depth_pool.py kept only the rows it could score, so the tool's own `not scored` line was dropped. The tool prints `depth 4: n/a (live 0/0 reached, offline 15/20) not scored` whenever one side has no denominator, and its reason for refusing to score it is written down in npu/mtp/mtp_head.h:96 -- at tau>0 the chain gate stops before depths 3-4, so that line is expected, not exotic. A pool over a four-depth run whose chains reached three depths therefore printed `3 of 3 scored depths within 2 pp at the same positions`: a bare full pass over a table that had silently lost a depth, which is the phantom-depth read (#916) one layer up. The same hole on the prompt axis. This lane runs one or two prompts per ticket, so prompts 0,1,5 is a normal pool; the header printed `prompts 0..5`, i.e. the range it spans rather than the set it measured, and the brief's window is prompts 0..7 -- so a gapped pool read as the whole window. Both are now reported rather than inferred: * a depth some ticket reported as not scored and no ticket scored a cell for is printed as unmeasured and named, and the verdict counts it: `3 of 3 scored depths ... (1 of 4 depths unmeasured: 4)`; * a non-contiguous prompt set says which prompts are missing instead of printing a range. Red on the old tool, green after -- tests/mtp_depth_pool_test.py grew two fixtures and reports: FAIL the not-scored depth is printed as unmeasured FAIL ... and it is depth 4, the one the log named FAIL the verdict says 3 of 4 depths, not a bare '3 of 3': 3 of 3 scored depths within 2 pp ... FAIL the header names the hole instead of spanning it: ... (prompts 0..5, 3 ticket(s)) (the rest of the suite, including the four refusal cases, was already green and stays green) The recorded 8x64 window is unchanged to the count: `--selftest` over the three clause55 tickets still pools 32 cells = 8 prompts x 4 depths at +0.00 / +0.55 / +0.09 / +1.08 pp, `4 of 4 scored depths`, PASS. --- tests/mtp_depth_pool_test.py | 44 ++++++++++++++++++++++++- tools/mtp_depth_pool.py | 64 ++++++++++++++++++++++++++++-------- 2 files changed, 94 insertions(+), 14 deletions(-) diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index 928cace95..84043b184 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -133,7 +133,49 @@ def run(args, logs): check("no denominator" in out5 and "1 of 1 scored depths" in out5, "the empty depth is reported as unmeasured and excluded from the scored count") -# 6. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +# 6. a depth the LOG reported as `not scored` is carried through as unmeasured, not dropped. +# +# The tool prints `depth 4: n/a (live 0/0 reached, offline 15/20) not scored` instead of a `%` row whenever one +# side has no denominator (npu/mtp/mtp_head.h:96), and at tau>0 the chain gate stops before depths 3-4. A pool +# that only kept the rows it could score dropped the depth entirely, so the table and the verdict line read as a +# complete pass over a table that had quietly lost a depth -- the phantom-depth read (#916) one layer up. +NOTSCORED = """prompt 0: TEXT IDENTICAL, 64 tokens, accepted 20/40, passes/token 0.500, stream 72 rows, 0 mismatch + per-depth p0: 1 18/20 2 13/20 3 2/20 4 0/20 + match p0: 20 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 90.00 % (18/20, reached) offline conditional 90.00 % (18/20) +depth 2: live 65.00 % (13/20, reached) offline conditional 65.00 % (13/20) +depth 3: live 10.00 % (2/20, reached) offline conditional 10.00 % (2/20) +depth 4: n/a (live 0/0 reached, offline 15/20) not scored +depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp PASS +depth 2: live 65.00 % (13/20) offline 65.00 % (13/20) delta +0.00 pp PASS +depth 3: live 10.00 % (2/20) offline 10.00 % (2/20) delta +0.00 pp PASS +depth 4: n/a (live 0/0 reached, offline 15/20) not scored +""" +rc6, out6, _ = run([], [("notscored.log", NOTSCORED)]) +check(rc6 == 0, "a log with a not-scored depth still pools") +check("unmeasured -- no chain reached this depth" in out6, "the not-scored depth is printed as unmeasured") +check("4 unmeasured" in out6, "... and it is depth 4, the one the log named") +check("3 of 3 scored depths" in out6 and "1 of 4 depths unmeasured: 4" in out6, + f"the verdict says 3 of 4 depths, not a bare '3 of 3': {out6.strip().splitlines()[-1]}") + +# 7. a pooled prompt set with a hole says so, instead of printing the range it spans. +# +# This lane runs one or two prompts per ticket, so prompts 0,1,5,6,7 is a normal pool -- but printed as +# "prompts 0..7" it reads as the brief's whole window. +def ticket(p): + return (f"prompt {p}: TEXT IDENTICAL, 64 tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch\n" + f"depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10)\n" + f"depth 1: live 80.00 % (8/10) offline 80.00 % (8/10) delta +0.00 pp PASS\n") + + +rc7, out7, _ = run([], [(f"p{p}.log", ticket(p)) for p in (0, 1, 5)]) +check(rc7 == 0, "a gapped prompt set pools without a refusal (per-prompt tickets are the normal case)") +check("NOT a contiguous window: 2,3,4 missing of 0..5" in out7, + f"the header names the hole instead of spanning it: {out7.splitlines()[0]}") +rc8, out8, _ = run([], [(f"q{p}.log", ticket(p)) for p in (0, 1, 2)]) +check("prompts 0..2" in out8 and "NOT a contiguous" not in out8, "a contiguous pool keeps the plain range form") + +# 8. the tool's own refusals, and (with logs) the recorded 8x64 known answer. r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py index 01e6f7bf5..2482e214e 100755 --- a/tools/mtp_depth_pool.py +++ b/tools/mtp_depth_pool.py @@ -28,6 +28,11 @@ # disagreement between two tickets (or a format change), never an average to be taken. # A ticket's own run-level row is the sum of its per-prompt rows, so it is checked too: a truncated log cannot # then pass silently as a smaller window. +# 3. a depth the tool reported as "n/a ... not scored" is carried through as UNMEASURED instead of being +# dropped, and a pooled prompt set that is not contiguous says so. Both are one failure: a pool that only +# reports the cells it scored can read as a complete pass over a window it never covered -- at tau>0 the +# chain gate stops before depths 3-4 (npu/mtp/mtp_head.h:96), and this lane's tickets are per-prompt slices, +# so a gap in either direction is normal and must be visible in the verdict. # # tools/mtp_depth_pool.py ticket-a.log ticket-b.log ... # pooled table + the 2 pp verdict per depth # tools/mtp_depth_pool.py --selftest # the refusals and the arithmetic, no logs needed @@ -47,6 +52,10 @@ # depth 2: live 52.78 % (38/72) offline 52.11 % (37/71) delta +0.67 pp PASS DETAIL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+), reached\)\s+" r"offline conditional\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)") +# The third shape: the tool refuses to score a depth one side never reached and says so on both the per-prompt and +# the run-level lines (npu/mtp/mtp_head.h:96 -- "0/0 is not agreement"): +# depth 4: n/a (live 0/0 reached, offline 15/20) not scored +NOTSCORED = re.compile(r"depth (\d+):\s+n/a \(live (\d+)/(\d+) reached, offline (\d+)/(\d+)\)") RUNLEVEL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+" r"offline\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+delta") PROMPT = re.compile(r"(?:per-depth p|prompt )(\d+)\b") @@ -57,12 +66,12 @@ def parse_ticket(path): - """Return {prompt, depth: (live_n, live_d, off_n, off_d)} for one ticket log. + """Return ({prompt, depth: (live_n, live_d, off_n, off_d)}, {depths the log says were not scored}). Raises SystemExit on a log it must not pool: a depth row before any prompt header (nothing to key it to), a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own run-level row (a short read).""" - cells, runlevel, prompt = {}, {}, None + cells, runlevel, prompt, not_scored = {}, {}, None, set() for ln in open(path, errors="replace").read().splitlines(): if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary break @@ -82,6 +91,10 @@ def parse_ticket(path): f"different counts {cells[key]} then {got}") cells.setdefault(key, got) continue + m = NOTSCORED.search(ln) + if m: + not_scored.add(int(m.group(1))) + continue m = RUNLEVEL.search(ln) if m: runlevel[int(m.group(1))] = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) @@ -95,14 +108,18 @@ def parse_ticket(path): if runlevel and len(runlevel) != len({d for _, d in cells}): raise SystemExit(f"mtp_depth_pool: {path}: run-level row covers {sorted(runlevel)} but per-prompt rows " f"cover {sorted({d for _, d in cells})}") - return cells + return cells, not_scored def pool(paths): - """Union the tickets' cells, refusing any disagreement on a key. Returns (cells, sources).""" - seen, sources = {}, 0 + """Union the tickets' cells, refusing any disagreement on a key. + + Returns (cells, unmeasured, sources); `unmeasured` holds the depths some ticket reported as `not scored` + that no ticket scored a cell for -- kept, so the pool cannot report a pass over a depth it never saw.""" + seen, unmeasured, sources = {}, set(), 0 for p in paths: - rows = parse_ticket(p) + rows, not_scored = parse_ticket(p) + unmeasured |= not_scored if not rows: print(f"mtp_depth_pool: {p}: no matched rows (was it run with --live --match?)", file=sys.stderr) continue @@ -112,15 +129,28 @@ def pool(paths): raise SystemExit(f"mtp_depth_pool: {p} disagrees with an earlier ticket on prompt {k[0]} " f"depth {k[1]}: {seen[k]} then {v}") seen.setdefault(k, v) - return seen, sources + return seen, unmeasured - {d for _, d in seen}, sources + + +def coverage(prompts): + """The pooled prompt set, saying out loud when it is a slice with holes rather than a contiguous window. + + A missing prompt is not a formatting detail: the brief's window is prompts 0..7, so a pool over 0,1,5,6,7 + printed as "prompts 0..7" reads as the whole window while five of eight prompts were measured.""" + lo, hi, have = prompts[0], prompts[-1], set(prompts) + missing = [p for p in range(lo, hi + 1) if p not in have] + if not missing: + return f"prompts {lo}..{hi}" + return (f"prompts {','.join(str(p) for p in prompts)} -- NOT a contiguous window: " + f"{','.join(str(p) for p in missing)} missing of {lo}..{hi}") -def table(seen, sources): +def table(seen, sources, unmeasured=()): """Print the pooled per-depth table; return (inside, scored, totals per depth).""" depths = sorted({d for _, d in seen}) prompts = sorted({p for p, _ in seen}) print(f"pooled {len(seen)} scored cells = {len(prompts)} prompts x {len(depths)} depths " - f"(prompts {prompts[0]}..{prompts[-1]}, {sources} ticket(s))") + f"({coverage(prompts)}, {sources} ticket(s))") print(f"\n{'depth':>5} {'live':>18} {'offline (same positions)':>24} {'delta':>9} verdict") inside = scored = 0 totals = {} @@ -142,7 +172,15 @@ def table(seen, sources): print(f"{d:>5} {f'{live_n}/{live_d} = {100.0 * live_n / live_d:.2f}%':>18} " f"{f'{off_n}/{off_d} = {100.0 * off_n / off_d:.2f}%':>24} {f'{delta:+.2f} pp':>9} " f"{'PASS' if ok else 'FAIL'}") - print(f"\n{inside} of {scored} scored depths within {BAR:g} pp at the same positions") + for d in sorted(unmeasured): + # Named, not omitted: the depth was asked for and no chain in any ticket reached it, so it is + # unmeasured -- and a table that simply lacks the row reads as a window that shrank to fit. + print(f"{d:>5} unmeasured -- no chain reached this depth in any ticket; not scored") + tail = "" + if unmeasured: + tail = (f" ({len(unmeasured)} of {len(depths) + len(unmeasured)} depths unmeasured: " + f"{','.join(str(d) for d in sorted(unmeasured))})") + print(f"\n{inside} of {scored} scored depths within {BAR:g} pp at the same positions{tail}") return inside, scored, totals @@ -189,7 +227,7 @@ def refusal_selftest(): fh.write("prompt 0: TEXT IDENTICAL\n" + d1 + run + tail) path = fh.name try: - cells = parse_ticket(path) + cells, _ = parse_ticket(path) if cells != {(0, 1): (8, 10, 8, 10)}: print(f"selftest FAIL tail repeat: parsed {cells}, wanted only the first row", file=sys.stderr) return 1 @@ -207,11 +245,11 @@ def main(argv): return refusal_selftest() print(__doc__) return 2 - seen, sources = pool(paths) + seen, unmeasured, sources = pool(paths) if not seen: print("mtp_depth_pool: nothing to pool", file=sys.stderr) return 1 - _, _, totals = table(seen, sources) + _, _, totals = table(seen, sources, unmeasured) if not selftest: return 0 bad = {d: (totals.get(d), RECORDED[d]) for d in RECORDED if totals.get(d) != RECORDED[d]} From 0c2afbc303897958a98faf23e56b09e2ad580fec Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:15:21 -0400 Subject: [PATCH 3/9] tools: the pooled verdict states the generation budget it covers, and refuses one prompt at two lengths Third hole of the same family as the previous commit: the pool reported which prompts and depths it scored, but never the generation budget, and the brief's window is 8 prompts x 64 tokens. The live per-prompt header carries it (`prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, ...`), so a pool mixing a 64-token ticket with a 32-token one silently produced a table whose shape matched neither -- the same failure as the prompt gap: a verdict that reads as the window it was not measured over. The budget is now part of the header (`..., 64 tokens/prompt, 3 ticket(s)`), and a pool that mixes budgets says so per prompt instead of inheriting whichever ticket was read first: MIXED token budgets: 64 on prompts 0; 128 on prompts 1 -- the pooled verdict is not one window Mixing lengths across *different* prompts stays legal, because a longer prompt is a legitimate measurement; the same prompt measured at two lengths is a contradiction about one cell and is refused, like any other disagreement already refused here. Only the live header shape is read: its `N tokens` is the emitted count (`r.stats.emitted`, i.e. --tokens), while the offline header's number in the same position is the stream length -- reading both would have compared two different quantities, which is the class of mistake this file exists to stop. Red on the previous commit, green after (three of the new checks fail against `9ec6cce8`, 26/26 pass now): FAIL a uniform budget is stated in the header FAIL the mixed budget is named per prompt FAIL prompt 3 at 64 and at 32 tokens is refused `--selftest` over the three clause55 tickets still reproduces the recorded window to the count, now labelled `64 tokens/prompt`: 32 cells = 8 prompts x 4 depths, +0.00 / +0.55 / +0.09 / +1.08 pp. --- tests/mtp_depth_pool_test.py | 19 +++++++++++--- tools/mtp_depth_pool.py | 50 ++++++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index 84043b184..09539927f 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -162,8 +162,8 @@ def run(args, logs): # # This lane runs one or two prompts per ticket, so prompts 0,1,5,6,7 is a normal pool -- but printed as # "prompts 0..7" it reads as the brief's whole window. -def ticket(p): - return (f"prompt {p}: TEXT IDENTICAL, 64 tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch\n" +def ticket(p, tokens=64): + return (f"prompt {p}: TEXT IDENTICAL, {tokens} tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch\n" f"depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10)\n" f"depth 1: live 80.00 % (8/10) offline 80.00 % (8/10) delta +0.00 pp PASS\n") @@ -172,10 +172,23 @@ def ticket(p): check(rc7 == 0, "a gapped prompt set pools without a refusal (per-prompt tickets are the normal case)") check("NOT a contiguous window: 2,3,4 missing of 0..5" in out7, f"the header names the hole instead of spanning it: {out7.splitlines()[0]}") +check("64 tokens/prompt" in out7, "a uniform budget is stated in the header") rc8, out8, _ = run([], [(f"q{p}.log", ticket(p)) for p in (0, 1, 2)]) check("prompts 0..2" in out8 and "NOT a contiguous" not in out8, "a contiguous pool keeps the plain range form") -# 8. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +# 8. the generation budget is part of the window: 8 prompts x 64 tokens is the brief's, so a pool that mixes +# budgets says so instead of inheriting the shape of whichever ticket it read first. +rc9, out9, _ = run([], [("t64.log", ticket(0)), ("t128.log", ticket(1, tokens=128))]) +check(rc9 == 0, "mixed budgets still pool (a longer prompt is a legitimate measurement)") +check("MIXED token budgets" in out9 and "64 on prompts 0" in out9 and "128 on prompts 1" in out9, + f"the mixed budget is named per prompt: {[l for l in out9.splitlines() if 'MIXED' in l]}") + +# 9. the same prompt at two lengths is a contradiction about the same cell, not a wider window. +rc10, _, err10 = run([], [("same64.log", ticket(3)), ("same32.log", ticket(3, tokens=32))]) +check(rc10 == 1 and "two lengths of the same prompt" in err10, + f"prompt 3 at 64 and at 32 tokens is refused: {err10.strip()[:110]}") + +# 10. the tool's own refusals, and (with logs) the recorded 8x64 known answer. r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py index 2482e214e..a8bdd2119 100755 --- a/tools/mtp_depth_pool.py +++ b/tools/mtp_depth_pool.py @@ -59,6 +59,11 @@ RUNLEVEL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+" r"offline\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+delta") PROMPT = re.compile(r"(?:per-depth p|prompt )(\d+)\b") +# The live per-prompt header carries the generation budget, and it is the only shape that carries it correctly: +# prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, passes/token 0.625, stream 72 rows, 0 mismatch +# (the offline header's "N tokens" at the same position is the stream length, not the budget, so it is not read +# here.) The brief's window is 8 prompts x 64 tokens, so a pool mixing budgets is not one window. +PROMPT_TOKENS = re.compile(r"prompt (\d+):\s+(?:TEXT IDENTICAL|TEXT DIVERGED),\s+(\d+) tokens,") # The numbers this tool must reproduce when it is pointed at the recorded window (TMTPLOOP-STATE.md, # 2026-11-05): live (num, den) and offline (num, den) per depth, over 8 prompts x 64 tokens. @@ -66,18 +71,21 @@ def parse_ticket(path): - """Return ({prompt, depth: (live_n, live_d, off_n, off_d)}, {depths the log says were not scored}). + """Return ({prompt, depth: counts}, {depths not scored}, {prompt: tokens emitted}) for one ticket log. Raises SystemExit on a log it must not pool: a depth row before any prompt header (nothing to key it to), a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own run-level row (a short read).""" - cells, runlevel, prompt, not_scored = {}, {}, None, set() + cells, runlevel, prompt, not_scored, budget = {}, {}, None, set(), {} for ln in open(path, errors="replace").read().splitlines(): if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary break m = PROMPT.search(ln) if m and "depth " not in ln: prompt = int(m.group(1)) + t = PROMPT_TOKENS.search(ln) + if t: + budget[prompt] = int(t.group(2)) continue m = DETAIL.search(ln) if m: @@ -108,7 +116,7 @@ def parse_ticket(path): if runlevel and len(runlevel) != len({d for _, d in cells}): raise SystemExit(f"mtp_depth_pool: {path}: run-level row covers {sorted(runlevel)} but per-prompt rows " f"cover {sorted({d for _, d in cells})}") - return cells, not_scored + return cells, not_scored, budget def pool(paths): @@ -117,9 +125,15 @@ def pool(paths): Returns (cells, unmeasured, sources); `unmeasured` holds the depths some ticket reported as `not scored` that no ticket scored a cell for -- kept, so the pool cannot report a pass over a depth it never saw.""" seen, unmeasured, sources = {}, set(), 0 + budget = {} for p in paths: - rows, not_scored = parse_ticket(p) + rows, not_scored, bud = parse_ticket(p) unmeasured |= not_scored + for pr, tk in bud.items(): + if pr in budget and budget[pr] != tk: + raise SystemExit(f"mtp_depth_pool: {p} ran prompt {pr} for {tk} tokens but an earlier ticket ran " + f"it for {budget[pr]} -- two lengths of the same prompt are not one window") + budget.setdefault(pr, tk) if not rows: print(f"mtp_depth_pool: {p}: no matched rows (was it run with --live --match?)", file=sys.stderr) continue @@ -129,7 +143,7 @@ def pool(paths): raise SystemExit(f"mtp_depth_pool: {p} disagrees with an earlier ticket on prompt {k[0]} " f"depth {k[1]}: {seen[k]} then {v}") seen.setdefault(k, v) - return seen, unmeasured - {d for _, d in seen}, sources + return seen, unmeasured - {d for _, d in seen}, sources, budget def coverage(prompts): @@ -145,12 +159,28 @@ def coverage(prompts): f"{','.join(str(p) for p in missing)} missing of {lo}..{hi}") -def table(seen, sources, unmeasured=()): +def token_coverage(budget, prompts): + """The generation budget the pool actually covers, per prompt, and an explicit note when it is not one value.""" + got = {p: budget[p] for p in prompts if p in budget} + if not got: + return "unknown tokens/prompt", None + vals = sorted(set(got.values())) + if len(vals) == 1: + return f"{vals[0]} tokens/prompt", None + groups = "; ".join(f"{v} on prompts {','.join(str(p) for p in prompts if got.get(p) == v)}" + for v in vals) + return "MIXED token budgets", f"MIXED token budgets: {groups} -- the pooled verdict is not one window" + + +def table(seen, sources, unmeasured=(), budget=None): """Print the pooled per-depth table; return (inside, scored, totals per depth).""" depths = sorted({d for _, d in seen}) prompts = sorted({p for p, _ in seen}) + tokens_note, mixed = token_coverage(budget or {}, prompts) print(f"pooled {len(seen)} scored cells = {len(prompts)} prompts x {len(depths)} depths " - f"({coverage(prompts)}, {sources} ticket(s))") + f"({coverage(prompts)}, {tokens_note}, {sources} ticket(s))") + if mixed: + print(mixed) print(f"\n{'depth':>5} {'live':>18} {'offline (same positions)':>24} {'delta':>9} verdict") inside = scored = 0 totals = {} @@ -227,7 +257,7 @@ def refusal_selftest(): fh.write("prompt 0: TEXT IDENTICAL\n" + d1 + run + tail) path = fh.name try: - cells, _ = parse_ticket(path) + cells, _, _ = parse_ticket(path) if cells != {(0, 1): (8, 10, 8, 10)}: print(f"selftest FAIL tail repeat: parsed {cells}, wanted only the first row", file=sys.stderr) return 1 @@ -245,11 +275,11 @@ def main(argv): return refusal_selftest() print(__doc__) return 2 - seen, unmeasured, sources = pool(paths) + seen, unmeasured, sources, budget = pool(paths) if not seen: print("mtp_depth_pool: nothing to pool", file=sys.stderr) return 1 - _, _, totals = table(seen, sources, unmeasured) + _, _, totals = table(seen, sources, unmeasured, budget) if not selftest: return 0 bad = {d: (totals.get(d), RECORDED[d]) for d in RECORDED if totals.get(d) != RECORDED[d]} From 56487b491229f073632d7dfe375da203d2a4e526 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:16:33 -0400 Subject: [PATCH 4/9] tools: a pre-`reached` log is refused for the reason it is actually refused Last night's corpus sweep (all 70 `mtp_*/ticket-*` logs in ~/wt/mtp-window, one at a time) turned up two logs this tool refuses with the wrong explanation: `ticket-match-p0-1-20261102-151015.log` and `ticket-match-p0-3-diffs-20261102-152047.log` (2 of the 16 that carry cells; the other 54 are offline runs with no `--match` rows and are skipped as such). They are REFUSED CORRECTLY -- their per-prompt rows are the pre-`reached` shape, depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp whose live rate is the MARGINAL one, not the rate conditional on reaching the depth that the current column reports, so pooling them would mix two different quantities. But the diagnostic said "the log was read incompletely, do not pool it", which is about truncation, and would send a reader to re-run a complete six-minute ticket. Now the two cases are distinguished by structure rather than guessed: the run-level rows live after the `== live vs offline at the SAME positions` header, so a `delta`-shaped row BEFORE that header is a per-prompt row in the old shape, and the refusal names it -- : 8 per-prompt row(s) in the pre-`reached` shape (live rate marginal, not conditional on reaching the depth) -- this log predates the column, so its cells are a different quantity; re-run it with a current build rather than pooling it (the counts are right: 8 = 2 prompts x 4 depths on the first log, 16 = 4 x 4 on the second). A log that really is short still gets the "read incompletely" refusal, unchanged. Corpus re-swept after the change: 14 pool, 54 skipped as offline-only, the same 2 refused -- and now for the right reason. Recorded 8x64 known answer unchanged to the count (`32 cells = 8 prompts x 4 depths`, `64 tokens/prompt`, +0.00 / +0.55 / +0.09 / +1.08 pp, `selftest PASS`). 28/28 checks in tests/mtp_depth_pool_test.py; ctest 1/1; copyright clean. --- tests/mtp_depth_pool_test.py | 18 +++++++++++++++++- tools/mtp_depth_pool.py | 21 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index 09539927f..605ff0d73 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -188,7 +188,23 @@ def ticket(p, tokens=64): check(rc10 == 1 and "two lengths of the same prompt" in err10, f"prompt 3 at 64 and at 32 tokens is refused: {err10.strip()[:110]}") -# 10. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +# 10. a log whose per-prompt rows predate the `reached` column is a DIFFERENT quantity, and says so: its live +# rate is marginal, not conditional, so "the log was read incompletely" would send a reader to re-run a complete +# ticket. (2 of the 16 cell-bearing logs in this lane's own corpus are this shape.) +OLD_SHAPE = """prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, passes/token 0.625, stream 72 rows, 0 mismatch + match p0: 20 offline chains at this loop's own positions (stream 72 rows) + depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp + depth 2: live 65.00 % (13/20) offline 65.00 % (13/20) delta +0.00 pp + == live vs offline at the SAME positions (the 2 pp bar): + depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp PASS + depth 2: live 65.00 % (13/20) offline 65.00 % (13/20) delta +0.00 pp PASS +""" +rc11, _, err11 = run([], [("oldshape.log", OLD_SHAPE)]) +check(rc11 == 1, "a pre-`reached` log is refused rather than pooled as if it were the current shape") +check("pre-`reached` shape" in err11 and "read incompletely" not in err11, + f"the refusal names the real cause, not a short read: {err11.strip()[-120:]}") + +# 11. the tool's own refusals, and (with logs) the recorded 8x64 known answer. r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py index a8bdd2119..6d16bd7c7 100755 --- a/tools/mtp_depth_pool.py +++ b/tools/mtp_depth_pool.py @@ -56,6 +56,13 @@ # the run-level lines (npu/mtp/mtp_head.h:96 -- "0/0 is not agreement"): # depth 4: n/a (live 0/0 reached, offline 15/20) not scored NOTSCORED = re.compile(r"depth (\d+):\s+n/a \(live (\d+)/(\d+) reached, offline (\d+)/(\d+)\)") + +# The summary section, and with it the run-level rows, starts here in every log that has per-prompt rows: +# == live vs offline at the SAME positions (the 2 pp bar): +# Before it, a `depth N: live .. (n/d) offline .. (n/d) delta ..` row is a PER-PROMPT row in the pre-`reached` +# shape (2026-11-02 and earlier), whose live rate is the MARGINAL one rather than the rate conditional on reaching +# the depth -- a different quantity from the current column, so such a log cannot be pooled at all. +SUMMARY = "== live vs offline at the SAME positions" RUNLEVEL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+" r"offline\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+delta") PROMPT = re.compile(r"(?:per-depth p|prompt )(\d+)\b") @@ -77,9 +84,13 @@ def parse_ticket(path): a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own run-level row (a short read).""" cells, runlevel, prompt, not_scored, budget = {}, {}, None, set(), {} + old_shape, summary = 0, False for ln in open(path, errors="replace").read().splitlines(): if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary break + if SUMMARY in ln: + summary = True + continue m = PROMPT.search(ln) if m and "depth " not in ln: prompt = int(m.group(1)) @@ -105,7 +116,17 @@ def parse_ticket(path): continue m = RUNLEVEL.search(ln) if m: + if not summary: + old_shape += 1 runlevel[int(m.group(1))] = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) + # A log whose per-prompt rows are all in the pre-`reached` shape cannot be pooled: its live rate is marginal + # and its offline rate unconditional, so calling that "the log was read incompletely" would send the reader to + # re-run a ticket that is complete -- 2 of the 16 cell-bearing logs in this lane's corpus are that shape. + if runlevel and not cells and old_shape: + raise SystemExit(f"mtp_depth_pool: {path}: {old_shape} per-prompt row(s) in the pre-`reached` shape (live " + f"rate marginal, not conditional on reaching the depth) -- this log predates the column, " + f"so its cells are a different quantity; re-run it with a current build rather than " + f"pooling it") # The ticket's own run-level row is the pool of its per-prompt rows, so a short read cannot pass as a # smaller window: the two have to agree before this log is allowed into the pool at all. for d, rl in runlevel.items(): From 4f53bf6182ccb6161efefff4d98a08f51607e503 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:50:52 -0400 Subject: [PATCH 5/9] tools: the pooled verdict states the chain width too, and names the prompts that ran wider Fifth axis of the same defect, and the last one the window has: k. Each `--live --match` row carries every depth the run drafted, so a k=6 ticket pooled with k=4 ones produces a SIX-depth table whose depths 5-6 have cells only on the k=6 prompts -- and the verdict then reads `6 of 6 scored depths within 2 pp at the same positions`, i.e. a pass over depths five of the tickets never attempted. On the brief's deliverable (a four-depth table) that is not the claim being made. The chain width is read from the rows themselves (the deepest depth a prompt has a row for, scored or not-scored, so a depth the chain never reached still counts toward k) and printed: pooled 32 scored cells = 8 prompts x 4 depths (prompts 0..7, 64 tokens/prompt, k=4, 3 ticket(s)) and when it is not one value, per prompt: MIXED chain width (k): 4 on prompts 0; 6 on prompts 1 -- the depths above 4 come from the wider prompts only, not from the whole pool Pooling different widths stays legal -- a k sweep is a legitimate measurement -- but the table no longer implies the whole pool reached those depths. A prompt measured twice resolves to the wider run, which is the one whose cells the table keeps. Red on `56487b49`, green after: `the differing chain width is named per prompt`, `... and says what it means for the table` and `a uniform chain width is stated once in the header` all fail there. Suite 34/34; `--selftest` over the three clause55 tickets still reproduces the recorded window to the count, now labelled `k=4`; `ctest -R mtp_depth_pool` 1/1; copyright clean. --- tests/mtp_depth_pool_test.py | 22 +++++++++++++- tools/mtp_depth_pool.py | 59 ++++++++++++++++++++++++++---------- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index 605ff0d73..466d711ce 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -204,7 +204,27 @@ def ticket(p, tokens=64): check("pre-`reached` shape" in err11 and "read incompletely" not in err11, f"the refusal names the real cause, not a short read: {err11.strip()[-120:]}") -# 11. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +# 12. k is part of the window too: a k=6 ticket pooled with k=4 ones builds a six-depth table whose depths 5-6 +# exist on the wider prompts only, and `N of N scored depths` then reads as a pass over depths the rest never tried. +def ticket_k(p, k): + body = [f"prompt {p}: TEXT IDENTICAL, 64 tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch"] + body += [f" depth {d}: live 50.00 % (10/20, reached) offline conditional 50.00 % (10/20)" for d in range(1, k + 1)] + body.append(" == live vs offline at the SAME positions (the 2 pp bar):") + body += [f" depth {d}: live 50.00 % (10/20) offline 50.00 % (10/20) delta +0.00 pp PASS" + for d in range(1, k + 1)] + return "\n".join(body) + "\n" + + +rc12, out12, _ = run([], [("k4.log", ticket_k(0, 4)), ("k6.log", ticket_k(1, 6))]) +check(rc12 == 0, "a pool of runs at different k still pools (the cells themselves are not in conflict)") +check("MIXED chain width (k)" in out12 and "6 on prompts 1" in out12 and "4 on prompts 0" in out12, + f"the differing chain width is named per prompt: {[l for l in out12.splitlines() if 'MIXED chain' in l]}") +check("the depths above 4 come from the wider prompts only" in out12, + "... and says what it means for the table (depths 5-6 are not the whole pool)") +rc13, out13, _ = run([], [("k4a.log", ticket_k(0, 4)), ("k4b.log", ticket_k(1, 4))]) +check("k=4" in out13 and "MIXED chain width" not in out13, "a uniform chain width is stated once in the header") + +# 13. the tool's own refusals, and (with logs) the recorded 8x64 known answer. r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py index 6d16bd7c7..c7a4f15b3 100755 --- a/tools/mtp_depth_pool.py +++ b/tools/mtp_depth_pool.py @@ -29,10 +29,13 @@ # A ticket's own run-level row is the sum of its per-prompt rows, so it is checked too: a truncated log cannot # then pass silently as a smaller window. # 3. a depth the tool reported as "n/a ... not scored" is carried through as UNMEASURED instead of being -# dropped, and a pooled prompt set that is not contiguous says so. Both are one failure: a pool that only -# reports the cells it scored can read as a complete pass over a window it never covered -- at tau>0 the -# chain gate stops before depths 3-4 (npu/mtp/mtp_head.h:96), and this lane's tickets are per-prompt slices, -# so a gap in either direction is normal and must be visible in the verdict. +# dropped, and the window is stated as it was measured rather than as it was asked for: a pooled prompt set +# that is not contiguous says which prompts are missing, the generation budget is printed (`64 tokens/prompt` +# or a MIXED line naming each length per prompt), and the chain width is printed (`k=4`, or which prompts ran +# wider). All of it is one failure: a pool that only reports the cells it scored can read as a complete pass +# over a window it never covered -- at tau>0 the chain gate stops before depths 3-4 (npu/mtp/mtp_head.h:96), +# this lane's tickets are per-prompt slices, and a k=6 ticket pooled with k=4 ones adds depths 5-6 that only +# its own prompts ever attempted. # # tools/mtp_depth_pool.py ticket-a.log ticket-b.log ... # pooled table + the 2 pp verdict per depth # tools/mtp_depth_pool.py --selftest # the refusals and the arithmetic, no logs needed @@ -84,7 +87,7 @@ def parse_ticket(path): a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own run-level row (a short read).""" cells, runlevel, prompt, not_scored, budget = {}, {}, None, set(), {} - old_shape, summary = 0, False + old_shape, summary, width = 0, False, {} for ln in open(path, errors="replace").read().splitlines(): if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary break @@ -104,6 +107,7 @@ def parse_ticket(path): raise SystemExit(f"mtp_depth_pool: {path}: a depth row before any prompt line -- " f"refusing to guess which prompt it belongs to") key = (prompt, int(m.group(1))) + width[prompt] = max(width.get(prompt, 0), key[1]) # the chain width k this prompt was drafted at got = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) if key in cells and cells[key] != got: raise SystemExit(f"mtp_depth_pool: {path}: prompt {key[0]} depth {key[1]} scored twice with " @@ -113,6 +117,8 @@ def parse_ticket(path): m = NOTSCORED.search(ln) if m: not_scored.add(int(m.group(1))) + if prompt is not None: + width[prompt] = max(width.get(prompt, 0), int(m.group(1))) continue m = RUNLEVEL.search(ln) if m: @@ -137,7 +143,7 @@ def parse_ticket(path): if runlevel and len(runlevel) != len({d for _, d in cells}): raise SystemExit(f"mtp_depth_pool: {path}: run-level row covers {sorted(runlevel)} but per-prompt rows " f"cover {sorted({d for _, d in cells})}") - return cells, not_scored, budget + return cells, not_scored, budget, width def pool(paths): @@ -146,10 +152,12 @@ def pool(paths): Returns (cells, unmeasured, sources); `unmeasured` holds the depths some ticket reported as `not scored` that no ticket scored a cell for -- kept, so the pool cannot report a pass over a depth it never saw.""" seen, unmeasured, sources = {}, set(), 0 - budget = {} + budget, width = {}, {} for p in paths: - rows, not_scored, bud = parse_ticket(p) + rows, not_scored, bud, wid = parse_ticket(p) unmeasured |= not_scored + for pr, wid_k in wid.items(): + width[pr] = max(width.get(pr, 0), wid_k) # a deeper second run of one prompt wins, as its cells do for pr, tk in bud.items(): if pr in budget and budget[pr] != tk: raise SystemExit(f"mtp_depth_pool: {p} ran prompt {pr} for {tk} tokens but an earlier ticket ran " @@ -164,7 +172,7 @@ def pool(paths): raise SystemExit(f"mtp_depth_pool: {p} disagrees with an earlier ticket on prompt {k[0]} " f"depth {k[1]}: {seen[k]} then {v}") seen.setdefault(k, v) - return seen, unmeasured - {d for _, d in seen}, sources, budget + return seen, unmeasured - {d for _, d in seen}, sources, budget, width def coverage(prompts): @@ -193,15 +201,34 @@ def token_coverage(budget, prompts): return "MIXED token budgets", f"MIXED token budgets: {groups} -- the pooled verdict is not one window" -def table(seen, sources, unmeasured=(), budget=None): +def width_coverage(width, prompts): + """The chain width (k) the pool actually covers, and an explicit note when it is not one value. + + k is part of the window, not a detail of one ticket: a k=6 ticket pooled with k=4 ones builds a six-depth table + whose depths 5-6 exist on the k=6 prompts only, and `N of N scored depths` then reads as a full pass over depths + the other tickets never attempted.""" + got = {p: width[p] for p in prompts if p in width} + if not got: + return "k unknown", None + vals = sorted(set(got.values())) + if len(vals) == 1: + return f"k={vals[0]}", None + groups = "; ".join(f"{v} on prompts {','.join(str(p) for p in prompts if got.get(p) == v)}" for v in vals) + return "MIXED k", (f"MIXED chain width (k): {groups} -- the depths above {vals[0]} come from the wider prompts " + f"only, not from the whole pool") + + +def table(seen, sources, unmeasured=(), budget=None, width=None): """Print the pooled per-depth table; return (inside, scored, totals per depth).""" depths = sorted({d for _, d in seen}) prompts = sorted({p for p, _ in seen}) tokens_note, mixed = token_coverage(budget or {}, prompts) + width_note, mixed_width = width_coverage(width or {}, prompts) print(f"pooled {len(seen)} scored cells = {len(prompts)} prompts x {len(depths)} depths " - f"({coverage(prompts)}, {tokens_note}, {sources} ticket(s))") - if mixed: - print(mixed) + f"({coverage(prompts)}, {tokens_note}, {width_note}, {sources} ticket(s))") + for note in (mixed, mixed_width): + if note: + print(note) print(f"\n{'depth':>5} {'live':>18} {'offline (same positions)':>24} {'delta':>9} verdict") inside = scored = 0 totals = {} @@ -278,7 +305,7 @@ def refusal_selftest(): fh.write("prompt 0: TEXT IDENTICAL\n" + d1 + run + tail) path = fh.name try: - cells, _, _ = parse_ticket(path) + cells, _, _, _ = parse_ticket(path) if cells != {(0, 1): (8, 10, 8, 10)}: print(f"selftest FAIL tail repeat: parsed {cells}, wanted only the first row", file=sys.stderr) return 1 @@ -296,11 +323,11 @@ def main(argv): return refusal_selftest() print(__doc__) return 2 - seen, unmeasured, sources, budget = pool(paths) + seen, unmeasured, sources, budget, width = pool(paths) if not seen: print("mtp_depth_pool: nothing to pool", file=sys.stderr) return 1 - _, _, totals = table(seen, sources, unmeasured, budget) + _, _, totals = table(seen, sources, unmeasured, budget, width) if not selftest: return 0 bad = {d: (totals.get(d), RECORDED[d]) for d in RECORDED if totals.get(d) != RECORDED[d]} From 69bce5ae61b006127953f01472be7c8d9338f0d0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:51:57 -0400 Subject: [PATCH 6/9] tools: the pooled verdict states the run identity (vocab, batch width, hidden source) Last input that can change what a cell means, and the one this lane has published the largest number for: the hidden source alone moved acceptance between 2.1 and 3.19 tokens/pass, and `mtp_acceptance` prints the identity of the run that produced a log on one line at init, forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm so a pool that mixes a `residual` ticket with `post-final-norm` ones averages two measurements while reporting one table. Same for a different vocabulary (a different model) and a different batch width (a different sealed ELF family). None of that is visible in the cells; all of it is in the log. The identity is now in the header -- pooled 32 scored cells = 8 prompts x 4 depths (prompts 0..7, 64 tokens/prompt, k=4, vocab 248320, max_batch_rows 16, hidden post-final-norm, 3 ticket(s)) -- and a mixed pool names each identity with the ticket it came from instead of blending them: MIXED run identity: vocab 248320/M16/post-final-norm from norm.log; vocab 248320/M16/residual from rsd.log -- a different model, batch width or hidden source is a different measurement, not one pool Only tickets that contributed cells are listed, so an offline log in the same command does not fake a mix. With this the header states the whole window -- prompt set (and any gap), depth coverage (and any depth no chain reached), generation budget, chain width, and run identity -- which is the set of inputs the logs carry that can change a cell. The lane's own corpus is uniform on all of them (86 logs: one model, vocab 248320, M=16, post-final-norm), so this changes no recorded number; it stops the next pool from quietly not being one. Red on `4f53bf61`, green after (2 checks, 36/36 total); `--selftest` still reproduces the recorded 8x64 window to the count; the corpus sweep is unchanged (14 pool / 54 offline-only / 2 refused for the pre-`reached` shape); `ctest -R mtp_depth_pool` 1/1; copyright clean. --- tests/mtp_depth_pool_test.py | 14 ++++++++++- tools/mtp_depth_pool.py | 49 ++++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index 466d711ce..5a5b28fc7 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -224,7 +224,19 @@ def ticket_k(p, k): rc13, out13, _ = run([], [("k4a.log", ticket_k(0, 4)), ("k4b.log", ticket_k(1, 4))]) check("k=4" in out13 and "MIXED chain width" not in out13, "a uniform chain width is stated once in the header") -# 13. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +# 14. the run identity (vocab, batch width, hidden source) is part of the window: 2.1 vs 3.19 tokens/pass on the +# hidden-source switch alone is more than the bar, so pooling across it would average two measurements. +READY_NORM = "forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm\n" +READY_RSD = "forward ready: vocab 248320, max_batch_rows 16, hidden source residual\n" +rc14, out14, _ = run([], [("norm.log", READY_NORM + ticket_k(0, 4)), ("rsd.log", READY_RSD + ticket_k(1, 4))]) +check(rc14 == 0, "a pool across hidden sources still pools (the cells themselves are not in conflict)") +check("MIXED run identity" in out14 and "post-final-norm" in out14 and "residual" in out14, + f"a differing hidden source is named rather than averaged: {[l for l in out14.splitlines() if 'MIXED run' in l]}") +rc15, out15, _ = run([], [("n1.log", READY_NORM + ticket_k(0, 4)), ("n2.log", READY_NORM + ticket_k(1, 4))]) +check("vocab 248320, max_batch_rows 16, hidden post-final-norm" in out15 and "MIXED run identity" not in out15, + "a uniform run identity is stated in the header") + +# 15. the tool's own refusals, and (with logs) the recorded 8x64 known answer. r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py index c7a4f15b3..329068b80 100755 --- a/tools/mtp_depth_pool.py +++ b/tools/mtp_depth_pool.py @@ -43,6 +43,7 @@ # # Exit: 0 the pool was computed (individual depths may still read FAIL), 1 a refusal, 2 no input. """tools/mtp_depth_pool.py -- the per-depth live-vs-offline verdict over several mtp_acceptance tickets.""" +import os import re import sys @@ -66,6 +67,13 @@ # shape (2026-11-02 and earlier), whose live rate is the MARGINAL one rather than the rate conditional on reaching # the depth -- a different quantity from the current column, so such a log cannot be pooled at all. SUMMARY = "== live vs offline at the SAME positions" + +# The run's own identity line, printed once per process at init: +# forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +# All three change what a cell means -- a different vocabulary is a different model, a different batch width is a +# different sealed ELF family, and the hidden source moves acceptance by more than the bar (this lane has measured +# 2.1 vs 3.19 tokens/pass on that one switch alone). A pool that mixes them is not one measurement. +IDENTITY = re.compile(r"forward ready:\s+vocab (\d+),\s+max_batch_rows (\d+),\s+hidden source (\S+)") RUNLEVEL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+" r"offline\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+delta") PROMPT = re.compile(r"(?:per-depth p|prompt )(\d+)\b") @@ -87,13 +95,17 @@ def parse_ticket(path): a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own run-level row (a short read).""" cells, runlevel, prompt, not_scored, budget = {}, {}, None, set(), {} - old_shape, summary, width = 0, False, {} + old_shape, summary, width, ident = 0, False, {}, None for ln in open(path, errors="replace").read().splitlines(): if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary break if SUMMARY in ln: summary = True continue + m = IDENTITY.search(ln) + if m: + ident = (int(m.group(1)), int(m.group(2)), m.group(3)) + continue m = PROMPT.search(ln) if m and "depth " not in ln: prompt = int(m.group(1)) @@ -143,7 +155,7 @@ def parse_ticket(path): if runlevel and len(runlevel) != len({d for _, d in cells}): raise SystemExit(f"mtp_depth_pool: {path}: run-level row covers {sorted(runlevel)} but per-prompt rows " f"cover {sorted({d for _, d in cells})}") - return cells, not_scored, budget, width + return cells, not_scored, budget, width, ident def pool(paths): @@ -152,12 +164,14 @@ def pool(paths): Returns (cells, unmeasured, sources); `unmeasured` holds the depths some ticket reported as `not scored` that no ticket scored a cell for -- kept, so the pool cannot report a pass over a depth it never saw.""" seen, unmeasured, sources = {}, set(), 0 - budget, width = {}, {} + budget, width, idents = {}, {}, {} for p in paths: - rows, not_scored, bud, wid = parse_ticket(p) + rows, not_scored, bud, wid, ident = parse_ticket(p) unmeasured |= not_scored for pr, wid_k in wid.items(): width[pr] = max(width.get(pr, 0), wid_k) # a deeper second run of one prompt wins, as its cells do + if ident: + idents.setdefault(ident, []).append(p) for pr, tk in bud.items(): if pr in budget and budget[pr] != tk: raise SystemExit(f"mtp_depth_pool: {p} ran prompt {pr} for {tk} tokens but an earlier ticket ran " @@ -172,7 +186,7 @@ def pool(paths): raise SystemExit(f"mtp_depth_pool: {p} disagrees with an earlier ticket on prompt {k[0]} " f"depth {k[1]}: {seen[k]} then {v}") seen.setdefault(k, v) - return seen, unmeasured - {d for _, d in seen}, sources, budget, width + return seen, unmeasured - {d for _, d in seen}, sources, budget, width, idents def coverage(prompts): @@ -218,15 +232,28 @@ def width_coverage(width, prompts): f"only, not from the whole pool") -def table(seen, sources, unmeasured=(), budget=None, width=None): +def ident_coverage(idents): + """The run identity the pool covers (vocab, batch width, hidden source), and a note when it is not one run.""" + if not idents: + return "run identity unstated", None + if len(idents) == 1: + v, m, h = next(iter(idents)) + return f"vocab {v}, max_batch_rows {m}, hidden {h}", None + parts = "; ".join(f"vocab {v}/M{m}/{h} from {os.path.basename(ps[0])}" for (v, m, h), ps in idents.items()) + return "MIXED run identity", (f"MIXED run identity: {parts} -- a different model, batch width or hidden source " + f"is a different measurement, not one pool") + + +def table(seen, sources, unmeasured=(), budget=None, width=None, idents=None): """Print the pooled per-depth table; return (inside, scored, totals per depth).""" depths = sorted({d for _, d in seen}) prompts = sorted({p for p, _ in seen}) tokens_note, mixed = token_coverage(budget or {}, prompts) width_note, mixed_width = width_coverage(width or {}, prompts) + ident_note, mixed_ident = ident_coverage(idents or {}) print(f"pooled {len(seen)} scored cells = {len(prompts)} prompts x {len(depths)} depths " - f"({coverage(prompts)}, {tokens_note}, {width_note}, {sources} ticket(s))") - for note in (mixed, mixed_width): + f"({coverage(prompts)}, {tokens_note}, {width_note}, {ident_note}, {sources} ticket(s))") + for note in (mixed, mixed_width, mixed_ident): if note: print(note) print(f"\n{'depth':>5} {'live':>18} {'offline (same positions)':>24} {'delta':>9} verdict") @@ -305,7 +332,7 @@ def refusal_selftest(): fh.write("prompt 0: TEXT IDENTICAL\n" + d1 + run + tail) path = fh.name try: - cells, _, _, _ = parse_ticket(path) + cells, _, _, _, _ = parse_ticket(path) if cells != {(0, 1): (8, 10, 8, 10)}: print(f"selftest FAIL tail repeat: parsed {cells}, wanted only the first row", file=sys.stderr) return 1 @@ -323,11 +350,11 @@ def main(argv): return refusal_selftest() print(__doc__) return 2 - seen, unmeasured, sources, budget, width = pool(paths) + seen, unmeasured, sources, budget, width, idents = pool(paths) if not seen: print("mtp_depth_pool: nothing to pool", file=sys.stderr) return 1 - _, _, totals = table(seen, sources, unmeasured, budget, width) + _, _, totals = table(seen, sources, unmeasured, budget, width, idents) if not selftest: return 0 bad = {d: (totals.get(d), RECORDED[d]) for d in RECORDED if totals.get(d) != RECORDED[d]} From 7d5f34c6f52ffb875e5c71e1ca8b45ebd8c6dca1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 17:20:42 -0400 Subject: [PATCH 7/9] tests: the recorded 8x64 window is now checkable in CI, not only on the box that holds the tickets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool's known answer -- the per-depth live-vs-offline table the brief is judged on -- could only be reproduced with the original `npu-hold` logs, which live in one home directory on one machine. So the artefact that produces the deliverable's published verdict could break its own recorded numbers and CI would stay green: `ctest mtp_depth_pool` ran only the refusal self-test, with no logs. The three tickets are now shipped as fixtures under `tests/fixtures/mtp_clause55/`, extracted verbatim from the 2026-11-05 tickets (prompts 0 / 1-4 / 5-7, 78 lines together), and the test runs `--selftest` over them: PASS the fixtures pool to the recorded 8 x 4 cells PASS fixture pool reports 136/181 = 75.14% PASS fixture pool reports 86/136 = 63.24% PASS fixture pool reports 63/86 = 73.26% PASS fixture pool reports 53/63 = 84.13% PASS all four depths are inside the bar on the fixtures PASS the tool's own known-answer check passes on them Nothing the tool reads was edited -- the `[ Ns]` stamps, the `[live joint …]` brackets and the `match pN:` headers are as the tickets printed them -- so the pool still has to reproduce the counts instead of a fixture written to agree with the code. The extraction command is in the directory's README, with the tickets it came from. Proven it is not vacuous: adding 1 to `live_n` in `table()` reds 8 checks, including `the shipped fixtures reproduce the recorded 8x64 window (rc=1)`; reverting returns `all checks passed` (44 checks). Copyright clean; no model, no device, no token. --- tests/fixtures/mtp_clause55/README.md | 23 +++++++++++++++++++++++ tests/mtp_depth_pool_test.py | 19 ++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/mtp_clause55/README.md diff --git a/tests/fixtures/mtp_clause55/README.md b/tests/fixtures/mtp_clause55/README.md new file mode 100644 index 000000000..4025e16f1 --- /dev/null +++ b/tests/fixtures/mtp_clause55/README.md @@ -0,0 +1,23 @@ +# The three tickets that produced the recorded 8x64 window + +These are the only in-tree inputs that let `tools/mtp_depth_pool.py --selftest` check its **known answer**: the +per-depth live-vs-offline table the brief is judged on (8 prompts x 64 tokens, k=4, tau=0, +`+0.00 / +0.55 / +0.09 / +1.08 pp` at depths 1-4, 75.14 / 63.24 / 73.26 / 84.13 %). Without them the check needs the +original ticket logs, which live in one home directory on one box — so CI could break the tool without noticing. + +**Provenance, so these are not a restatement of the answer.** The three `.log` files are the lines the parser reads, +extracted verbatim from the `npu-hold` tickets of 2026-11-05 (the same tickets whose numbers the lane recorded): + +```sh +grep -E "forward ready:|prompt [0-9]+:|depth [0-9]+: |match p[0-9]+:|== live vs offline at the SAME|== ticket " \ + ticket-clause55-p0-20261105-115131.log > p0.log # 15 lines, prompts 0 +grep -E "..." ticket-clause55-p1-4-20261105-115551.log > p1-4.log # 35 lines, prompts 1-4 +grep -E "..." ticket-clause55-p5-7-20261105-120713.log > p5-7.log # 28 lines, prompts 5-7 +``` + +Everything the tool ignores is dropped — the engine chatter, the ELF loads, the per-prompt `per-depth pN:` compact +lines, the phase split. Nothing the tool reads is edited: the `[ Ns]` elapsed stamps, the `[live joint …]` brackets +and the `match pN:` headers are exactly as the tickets printed them, and the pool still has to reproduce the recorded +counts from them (that is what makes it a known answer rather than a fixture written to agree with the code). + +`tests/mtp_depth_pool_test.py` runs `--selftest` over these three and fails if the pooled cells drift by one count. diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index 5a5b28fc7..c24f72cde 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -236,7 +236,24 @@ def ticket_k(p, k): check("vocab 248320, max_batch_rows 16, hidden post-final-norm" in out15 and "MIXED run identity" not in out15, "a uniform run identity is stated in the header") -# 15. the tool's own refusals, and (with logs) the recorded 8x64 known answer. +# 15. the recorded 8x64 window, from the three tickets shipped under tests/fixtures/mtp_clause55 (see its README). +# Without this the known answer is only checkable on the machine that holds the original `npu-hold` logs, i.e. the tool +# that produces the deliverable's published verdict could break it and CI would not notice. +FIXTURES = os.path.join(ROOT, "tests", "fixtures", "mtp_clause55") +CL = [os.path.join(FIXTURES, n) for n in ("p0.log", "p1-4.log", "p5-7.log")] +if all(os.path.exists(p) for p in CL): + r16 = subprocess.run([sys.executable, TOOL, "--selftest"] + CL, capture_output=True, text=True) + check(r16.returncode == 0, f"the shipped fixtures reproduce the recorded 8x64 window (rc={r16.returncode})") + check("32 scored cells = 8 prompts x 4 depths" in r16.stdout, "the fixtures pool to the recorded 8 x 4 cells") + for want in ("136/181 = 75.14%", "86/136 = 63.24%", "63/86 = 73.26%", "53/63 = 84.13%"): + check(want in r16.stdout, f"fixture pool reports {want}") + check("4 of 4 scored depths within 2 pp" in r16.stdout, "all four depths are inside the bar on the fixtures") + check("selftest PASS: 4 depths reproduce the recorded 8x64 window" in r16.stdout, + "the tool's own known-answer check passes on them") +else: + check(False, f"tests/fixtures/mtp_clause55 is incomplete: {CL}") + +# 16. the tool's own refusals, and (no logs) nothing else: this is what runs on a box without the ticket corpus. r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") From 673faa2ecc5cb25c3645e69ab6dcce5ae2e600b1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 17:21:08 -0400 Subject: [PATCH 8/9] tests: carry the Apache notice on the fixture README (copyright --check) --- tests/fixtures/mtp_clause55/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/fixtures/mtp_clause55/README.md b/tests/fixtures/mtp_clause55/README.md index 4025e16f1..78b9986a8 100644 --- a/tests/fixtures/mtp_clause55/README.md +++ b/tests/fixtures/mtp_clause55/README.md @@ -1,3 +1,19 @@ + # The three tickets that produced the recorded 8x64 window These are the only in-tree inputs that let `tools/mtp_depth_pool.py --selftest` check its **known answer**: the From 3880aef8687863ef2db77d78eb137105334322ea Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 17:29:07 -0400 Subject: [PATCH 9/9] tests: the clause-55 fixtures were never committed (.gitignore `*.log`), so CI could not read them The build on `673faa2e` failed in exactly one test, and it was mine: 53 - mtp_depth_pool (Failed) FAIL tests/fixtures/mtp_clause55 is incomplete: ['/__w/engine/engine/tests/fixtures/mtp_clause55/p0.log', ...] `.gitignore:19` is `*.log`, so `git add -A tests/fixtures/mtp_clause55` committed the README and silently left all three tickets behind. Locally the test passed because the untracked files were lying in the working tree; in a clean checkout there was nothing to read. The guard fired -- that is the guard doing its job, one environment too late -- but the real defect is that a fixture can be present locally and absent from the commit at all. Two changes, so this cannot recur quietly: * the fixtures carry `.txt` (the ignore rules do not match it) and the directory README says why, with the CI failure quoted, because a rename or a fresh clone must not be able to drop them; * `tests/mtp_depth_pool_test.py` now asserts with `git ls-files` that all three are **tracked** before it pools them, so an unfixtured tree fails locally rather than only on a clean checkout. Measured: with the files present but unstaged, `FAIL the three fixtures are tracked by git (0 of 3)`; staged, `PASS ... (3 of 3)` and `all checks passed`. Verified from a clean tree rather than in my working copy: a fresh `git worktree add` at this commit runs the test green, which is what CI will do. No model, no device, no token. --- tests/fixtures/mtp_clause55/README.md | 16 ++++++++---- tests/fixtures/mtp_clause55/p0.txt | 15 ++++++++++++ tests/fixtures/mtp_clause55/p1-4.txt | 35 +++++++++++++++++++++++++++ tests/fixtures/mtp_clause55/p5-7.txt | 28 +++++++++++++++++++++ tests/mtp_depth_pool_test.py | 10 +++++++- tools/copyright.py | 4 +++ 6 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/mtp_clause55/p0.txt create mode 100644 tests/fixtures/mtp_clause55/p1-4.txt create mode 100644 tests/fixtures/mtp_clause55/p5-7.txt diff --git a/tests/fixtures/mtp_clause55/README.md b/tests/fixtures/mtp_clause55/README.md index 78b9986a8..f079aa7bd 100644 --- a/tests/fixtures/mtp_clause55/README.md +++ b/tests/fixtures/mtp_clause55/README.md @@ -21,16 +21,22 @@ per-depth live-vs-offline table the brief is judged on (8 prompts x 64 tokens, k `+0.00 / +0.55 / +0.09 / +1.08 pp` at depths 1-4, 75.14 / 63.24 / 73.26 / 84.13 %). Without them the check needs the original ticket logs, which live in one home directory on one box — so CI could break the tool without noticing. -**Provenance, so these are not a restatement of the answer.** The three `.log` files are the lines the parser reads, +**Provenance, so these are not a restatement of the answer.** The three `.txt` files are the lines the parser reads, extracted verbatim from the `npu-hold` tickets of 2026-11-05 (the same tickets whose numbers the lane recorded): ```sh -grep -E "forward ready:|prompt [0-9]+:|depth [0-9]+: |match p[0-9]+:|== live vs offline at the SAME|== ticket " \ - ticket-clause55-p0-20261105-115131.log > p0.log # 15 lines, prompts 0 -grep -E "..." ticket-clause55-p1-4-20261105-115551.log > p1-4.log # 35 lines, prompts 1-4 -grep -E "..." ticket-clause55-p5-7-20261105-120713.log > p5-7.log # 28 lines, prompts 5-7 +pat="forward ready:|prompt [0-9]+:|depth [0-9]+: |match p[0-9]+:|== live vs offline at the SAME|== ticket " +grep -E "$pat" ticket-clause55-p0-20261105-115131.log > p0.txt # 15 lines, prompt 0 +grep -E "$pat" ticket-clause55-p1-4-20261105-115551.log > p1-4.txt # 35 lines, prompts 1-4 +grep -E "$pat" ticket-clause55-p5-7-20261105-120713.log > p5-7.txt # 28 lines, prompts 5-7 ``` +**Why `.txt` and not `.log`**: `.gitignore:19` is `*.log`, so the first cut of this directory committed the README and +silently left all three tickets behind — the test then passed on the machine that had them lying around and failed in CI +with `tests/fixtures/mtp_clause55 is incomplete`. A fixture that can be dropped by a rename or a re-clone is worse than +one with a different suffix, so the files carry `.txt` and `tests/mtp_depth_pool_test.py` asserts with `git ls-files` +that all three are **tracked**, which fails locally instead of only in CI. + Everything the tool ignores is dropped — the engine chatter, the ELF loads, the per-prompt `per-depth pN:` compact lines, the phase split. Nothing the tool reads is edited: the `[ Ns]` elapsed stamps, the `[live joint …]` brackets and the `match pN:` headers are exactly as the tickets printed them, and the pool still has to reproduce the recorded diff --git a/tests/fixtures/mtp_clause55/p0.txt b/tests/fixtures/mtp_clause55/p0.txt new file mode 100644 index 000000000..3a5db4c84 --- /dev/null +++ b/tests/fixtures/mtp_clause55/p0.txt @@ -0,0 +1,15 @@ +== ticket 'clause55 p0' 20261105-115131 hold=16min argv: --live --match --prompts 1 --prompt-base 0 --tokens 64 --k 4 --tau 0 +[ 169s] forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +[ 252s] prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, passes/token 0.625, stream 72 rows, 0 mismatch +[ 258s] match p0: 20 offline chains at this loop's own positions (stream 72 rows) +[ 258s] depth 1: live 90.00 % (18/20, reached) offline conditional 90.00 % (18/20) delta +0.00 pp [live joint 90.00 %, offline marginal 90.00 %] +[ 258s] depth 2: live 72.22 % (13/18, reached) offline conditional 72.22 % (13/18) delta +0.00 pp [live joint 65.00 %, offline marginal 65.00 %] +[ 258s] depth 3: live 53.85 % (7/13, reached) offline conditional 53.85 % (7/13) delta +0.00 pp [live joint 35.00 %, offline marginal 35.00 %] +[ 258s] depth 4: live 85.71 % (6/7, reached) offline conditional 85.71 % (6/7) delta +0.00 pp [live joint 30.00 %, offline marginal 35.00 %] +[ 258s] first verdict difference on the same token: stream position 13, depth 4: live reject, offline accept +[ 258s] == live vs offline at the SAME positions, per-depth acceptance CONDITIONAL on reaching the depth (the 2 pp bar): +[ 258s] depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp PASS +[ 258s] depth 2: live 72.22 % (13/18) offline 72.22 % (13/18) delta +0.00 pp PASS +[ 258s] depth 3: live 53.85 % (7/13) offline 53.85 % (7/13) delta +0.00 pp PASS +[ 258s] depth 4: live 85.71 % (6/7) offline 85.71 % (6/7) delta +0.00 pp PASS +== ticket 'clause55 p0' exited 0; log: /home/bcloud/wt/mtp-window/ticket-clause55-p0-20261105-115131.log diff --git a/tests/fixtures/mtp_clause55/p1-4.txt b/tests/fixtures/mtp_clause55/p1-4.txt new file mode 100644 index 000000000..1c9778ddf --- /dev/null +++ b/tests/fixtures/mtp_clause55/p1-4.txt @@ -0,0 +1,35 @@ +== ticket 'clause55 p1-4' 20261105-115551 hold=16min argv: --live --match --prompts 4 --prompt-base 1 --tokens 64 --k 4 --tau 0 +[ 66s] forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +[ 177s] prompt 1: TEXT IDENTICAL, 64 tokens, accepted 28/140, passes/token 1.109, stream 72 rows, 0 mismatch +[ 190s] match p1: 35 offline chains at this loop's own positions (stream 72 rows) +[ 190s] depth 1: live 60.00 % (21/35, reached) offline conditional 60.00 % (21/35) delta +0.00 pp [live joint 60.00 %, offline marginal 60.00 %] +[ 190s] depth 2: live 19.05 % (4/21, reached) offline conditional 19.05 % (4/21) delta +0.00 pp [live joint 11.43 %, offline marginal 22.86 %] +[ 191s] depth 3: live 75.00 % (3/4, reached) offline conditional 75.00 % (3/4) delta +0.00 pp [live joint 8.57 %, offline marginal 26.47 %] +[ 191s] depth 4: live 0.00 % (0/3, reached) offline conditional 0.00 % (0/3) delta +0.00 pp [live joint 0.00 %, offline marginal 8.82 %] +[ 191s] first verdict difference on the same token: stream position 9, depth 2: live reject, offline accept +[ 302s] prompt 2: TEXT IDENTICAL, 64 tokens, accepted 36/112, passes/token 0.875, stream 72 rows, 0 mismatch +[ 313s] match p2: 28 offline chains at this loop's own positions (stream 72 rows) +[ 313s] depth 1: live 60.71 % (17/28, reached) offline conditional 60.71 % (17/28) delta +0.00 pp [live joint 60.71 %, offline marginal 60.71 %] +[ 313s] depth 2: live 64.71 % (11/17, reached) offline conditional 64.71 % (11/17) delta +0.00 pp [live joint 39.29 %, offline marginal 42.86 %] +[ 313s] depth 3: live 45.45 % (5/11, reached) offline conditional 50.00 % (5/10) delta -4.55 pp > 2 pp [live joint 17.86 %, offline marginal 25.93 %] +[ 313s] depth 4: live 60.00 % (3/5, reached) offline conditional 60.00 % (3/5) delta +0.00 pp [live joint 10.71 %, offline marginal 19.23 %] +[ 313s] first verdict difference on the same token: stream position 13, depth 2: live reject, offline accept +[ 407s] prompt 3: TEXT IDENTICAL, 64 tokens, accepted 50/64, passes/token 0.500, stream 72 rows, 0 mismatch +[ 414s] match p3: 16 offline chains at this loop's own positions (stream 72 rows) +[ 414s] depth 1: live 87.50 % (14/16, reached) offline conditional 87.50 % (14/16) delta +0.00 pp [live joint 87.50 %, offline marginal 87.50 %] +[ 414s] depth 2: live 85.71 % (12/14, reached) offline conditional 85.71 % (12/14) delta +0.00 pp [live joint 75.00 %, offline marginal 75.00 %] +[ 414s] depth 3: live 100.00 % (12/12, reached) offline conditional 100.00 % (11/11) delta +0.00 pp [live joint 75.00 %, offline marginal 73.33 %] +[ 414s] depth 4: live 100.00 % (12/12, reached) offline conditional 100.00 % (11/11) delta +0.00 pp [live joint 75.00 %, offline marginal 80.00 %] +[ 414s] first verdict difference on the same token: stream position 9, depth 4: live reject, offline accept +[ 494s] prompt 4: TEXT IDENTICAL, 64 tokens, accepted 41/104, passes/token 0.812, stream 72 rows, 0 mismatch +[ 503s] match p4: 26 offline chains at this loop's own positions (stream 72 rows) +[ 503s] depth 1: live 76.92 % (20/26, reached) offline conditional 76.92 % (20/26) delta +0.00 pp [live joint 76.92 %, offline marginal 76.92 %] +[ 503s] depth 2: live 55.00 % (11/20, reached) offline conditional 52.63 % (10/19) delta +2.37 pp > 2 pp [live joint 42.31 %, offline marginal 40.00 %] +[ 503s] depth 3: live 54.55 % (6/11, reached) offline conditional 50.00 % (5/10) delta +4.55 pp > 2 pp [live joint 23.08 %, offline marginal 20.00 %] +[ 503s] depth 4: live 66.67 % (4/6, reached) offline conditional 60.00 % (3/5) delta +6.67 pp > 2 pp [live joint 15.38 %, offline marginal 12.50 %] +[ 503s] == live vs offline at the SAME positions, per-depth acceptance CONDITIONAL on reaching the depth (the 2 pp bar): +[ 503s] depth 1: live 68.57 % (72/105) offline 68.57 % (72/105) delta +0.00 pp PASS +[ 503s] depth 2: live 52.78 % (38/72) offline 52.11 % (37/71) delta +0.67 pp PASS +[ 503s] depth 3: live 68.42 % (26/38) offline 68.57 % (24/35) delta -0.15 pp PASS +[ 503s] depth 4: live 73.08 % (19/26) offline 70.83 % (17/24) delta +2.24 pp FAIL +== ticket 'clause55 p1-4' exited 0; log: /home/bcloud/wt/mtp-window/ticket-clause55-p1-4-20261105-115551.log diff --git a/tests/fixtures/mtp_clause55/p5-7.txt b/tests/fixtures/mtp_clause55/p5-7.txt new file mode 100644 index 000000000..3a4063e1e --- /dev/null +++ b/tests/fixtures/mtp_clause55/p5-7.txt @@ -0,0 +1,28 @@ +== ticket 'clause55 p5-7' 20261105-120713 hold=16min argv: --live --match --prompts 3 --prompt-base 5 --tokens 64 --k 4 --tau 0 +[ 49s] forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +[ 110s] prompt 5: TEXT IDENTICAL, 64 tokens, accepted 48/60, passes/token 0.484, stream 72 rows, 0 mismatch +[ 114s] match p5: 15 offline chains at this loop's own positions (stream 72 rows) +[ 114s] depth 1: live 86.67 % (13/15, reached) offline conditional 86.67 % (13/15) delta +0.00 pp [live joint 86.67 %, offline marginal 86.67 %] +[ 114s] depth 2: live 92.31 % (12/13, reached) offline conditional 92.31 % (12/13) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 114s] depth 3: live 100.00 % (12/12, reached) offline conditional 100.00 % (12/12) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 114s] depth 4: live 91.67 % (11/12, reached) offline conditional 91.67 % (11/12) delta +0.00 pp [live joint 73.33 %, offline marginal 80.00 %] +[ 114s] first verdict difference on the same token: stream position 9, depth 4: live reject, offline accept +[ 165s] prompt 6: TEXT IDENTICAL, 64 tokens, accepted 50/60, passes/token 0.469, stream 72 rows, 0 mismatch +[ 169s] match p6: 15 offline chains at this loop's own positions (stream 72 rows) +[ 169s] depth 1: live 93.33 % (14/15, reached) offline conditional 93.33 % (14/15) delta +0.00 pp [live joint 93.33 %, offline marginal 93.33 %] +[ 169s] depth 2: live 85.71 % (12/14, reached) offline conditional 85.71 % (12/14) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 169s] depth 3: live 100.00 % (12/12, reached) offline conditional 100.00 % (12/12) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 169s] depth 4: live 100.00 % (12/12, reached) offline conditional 100.00 % (11/11) delta +0.00 pp [live joint 80.00 %, offline marginal 78.57 %] +[ 231s] prompt 7: TEXT IDENTICAL, 64 tokens, accepted 41/104, passes/token 0.812, stream 72 rows, 0 mismatch +[ 237s] match p7: 26 offline chains at this loop's own positions (stream 72 rows) +[ 237s] depth 1: live 73.08 % (19/26, reached) offline conditional 73.08 % (19/26) delta +0.00 pp [live joint 73.08 %, offline marginal 73.08 %] +[ 237s] depth 2: live 57.89 % (11/19, reached) offline conditional 55.56 % (10/18) delta +2.34 pp > 2 pp [live joint 42.31 %, offline marginal 44.00 %] +[ 237s] depth 3: live 54.55 % (6/11, reached) offline conditional 50.00 % (5/10) delta +4.55 pp > 2 pp [live joint 23.08 %, offline marginal 28.00 %] +[ 237s] depth 4: live 83.33 % (5/6, reached) offline conditional 80.00 % (4/5) delta +3.33 pp > 2 pp [live joint 19.23 %, offline marginal 20.00 %] +[ 237s] first verdict difference on the same token: stream position 11, depth 2: live reject, offline accept +[ 237s] == live vs offline at the SAME positions, per-depth acceptance CONDITIONAL on reaching the depth (the 2 pp bar): +[ 237s] depth 1: live 82.14 % (46/56) offline 82.14 % (46/56) delta +0.00 pp PASS +[ 237s] depth 2: live 76.09 % (35/46) offline 75.56 % (34/45) delta +0.53 pp PASS +[ 237s] depth 3: live 85.71 % (30/35) offline 85.29 % (29/34) delta +0.42 pp PASS +[ 237s] depth 4: live 93.33 % (28/30) offline 92.86 % (26/28) delta +0.48 pp PASS +== ticket 'clause55 p5-7' exited 0; log: /home/bcloud/wt/mtp-window/ticket-clause55-p5-7-20261105-120713.log diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py index c24f72cde..6ec01876f 100755 --- a/tests/mtp_depth_pool_test.py +++ b/tests/mtp_depth_pool_test.py @@ -240,7 +240,15 @@ def ticket_k(p, k): # Without this the known answer is only checkable on the machine that holds the original `npu-hold` logs, i.e. the tool # that produces the deliverable's published verdict could break it and CI would not notice. FIXTURES = os.path.join(ROOT, "tests", "fixtures", "mtp_clause55") -CL = [os.path.join(FIXTURES, n) for n in ("p0.log", "p1-4.log", "p5-7.log")] +CL = [os.path.join(FIXTURES, n) for n in ("p0.txt", "p1-4.txt", "p5-7.txt")] +# A fixture that is present on the machine but not COMMITTED is the worse failure mode: the test passes locally and the +# CI checkout has nothing to read. It happened with `.log` (`.gitignore:19`), so being tracked is checked here when the +# tree is a git checkout, and the files carry a suffix the ignore rules do not match. +if os.path.isdir(os.path.join(ROOT, ".git")) or os.path.exists(os.path.join(ROOT, ".git")): + listed = subprocess.run(["git", "-C", ROOT, "ls-files", "--"] + [os.path.relpath(p, ROOT) for p in CL], + capture_output=True, text=True) + tracked = len([l for l in listed.stdout.splitlines() if l.strip()]) + check(tracked == len(CL), f"the three fixtures are tracked by git ({tracked} of {len(CL)})") if all(os.path.exists(p) for p in CL): r16 = subprocess.run([sys.executable, TOOL, "--selftest"] + CL, capture_output=True, text=True) check(r16.returncode == 0, f"the shipped fixtures reproduce the recorded 8x64 window (rc={r16.returncode})") diff --git a/tools/copyright.py b/tools/copyright.py index bd0391afa..9f47afa08 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -52,6 +52,10 @@ # engine#315 evidence artefacts that cannot carry a comment: diffs must stay byte-applicable # and data files have no comment syntax. Matches the existing "patches/*" exemption. "*.patch", "*.csv", "*.cfg", + # engine#1146: the clause-55 acceptance tickets are DATA a parser reads byte-verbatim (like + # tests/golden/*/tokens.txt), so a notice comment would make the fixture no longer the log it claims + # to be; the directory README carries the notice instead + "tests/fixtures/mtp_clause55/*.txt", # os/v2 packages: dpkg control files, apt/dpkg list files and the console banner are data # with no comment syntax - the same exemption patches/* and *.cfg get "os/v2/debian/*/control", "os/v2/debian/*/conffiles", "*.list",