diff --git a/CMakeLists.txt b/CMakeLists.txt index c2ee15796..9975caf17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -566,6 +566,11 @@ if(Python3_Interpreter_FOUND) # WIDER tile grid than the model's dimensions (the FastFlowLM repack pads intermediate_size to 512 columns, IM # 8960 -> 9216) instead of reporting it as an unreadable reshape. Pure python + numpy, no model, no device. add_test(NAME q4nx_diff_grid COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/q4nx_diff_grid_test.py) + # The pooled per-depth live-vs-offline verdict (tools/mtp_depth_pool.py): the brief's "within 2 pp at every + # depth" is judged on 8 prompts x 64 tokens POOLED, which is two or three npu-hold tickets of <= 3 min, and + # pooling those logs by eye produced two wrong numbers in this lane. Fixture tickets only -- no model, no + # device; the recorded 8x64 window is the tool's known-answer check and is run by hand with --selftest. + add_test(NAME mtp_depth_pool COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/mtp_depth_pool_test.py) # the t2 chunk kind (docs/npu.md "The t2 chunk kind"): synthetic only, because the real-model # leg needs 14 GB of Bonsai-2 images. Run tests/q4nx_t2_test.py without --no-model for the gate. add_test(NAME q4nx_t2 COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/q4nx_t2_test.py --no-model) diff --git a/tests/fixtures/mtp_clause55/README.md b/tests/fixtures/mtp_clause55/README.md new file mode 100644 index 000000000..f079aa7bd --- /dev/null +++ b/tests/fixtures/mtp_clause55/README.md @@ -0,0 +1,45 @@ + +# The three tickets that produced the recorded 8x64 window + +These are the only in-tree inputs that let `tools/mtp_depth_pool.py --selftest` check its **known answer**: the +per-depth live-vs-offline table the brief is judged on (8 prompts x 64 tokens, k=4, tau=0, +`+0.00 / +0.55 / +0.09 / +1.08 pp` at depths 1-4, 75.14 / 63.24 / 73.26 / 84.13 %). Without them the check needs the +original ticket logs, which live in one home directory on one box — so CI could break the tool without noticing. + +**Provenance, so these are not a restatement of the answer.** The three `.txt` files are the lines the parser reads, +extracted verbatim from the `npu-hold` tickets of 2026-11-05 (the same tickets whose numbers the lane recorded): + +```sh +pat="forward ready:|prompt [0-9]+:|depth [0-9]+: |match p[0-9]+:|== live vs offline at the SAME|== ticket " +grep -E "$pat" ticket-clause55-p0-20261105-115131.log > p0.txt # 15 lines, prompt 0 +grep -E "$pat" ticket-clause55-p1-4-20261105-115551.log > p1-4.txt # 35 lines, prompts 1-4 +grep -E "$pat" ticket-clause55-p5-7-20261105-120713.log > p5-7.txt # 28 lines, prompts 5-7 +``` + +**Why `.txt` and not `.log`**: `.gitignore:19` is `*.log`, so the first cut of this directory committed the README and +silently left all three tickets behind — the test then passed on the machine that had them lying around and failed in CI +with `tests/fixtures/mtp_clause55 is incomplete`. A fixture that can be dropped by a rename or a re-clone is worse than +one with a different suffix, so the files carry `.txt` and `tests/mtp_depth_pool_test.py` asserts with `git ls-files` +that all three are **tracked**, which fails locally instead of only in CI. + +Everything the tool ignores is dropped — the engine chatter, the ELF loads, the per-prompt `per-depth pN:` compact +lines, the phase split. Nothing the tool reads is edited: the `[ Ns]` elapsed stamps, the `[live joint …]` brackets +and the `match pN:` headers are exactly as the tickets printed them, and the pool still has to reproduce the recorded +counts from them (that is what makes it a known answer rather than a fixture written to agree with the code). + +`tests/mtp_depth_pool_test.py` runs `--selftest` over these three and fails if the pooled cells drift by one count. diff --git a/tests/fixtures/mtp_clause55/p0.txt b/tests/fixtures/mtp_clause55/p0.txt new file mode 100644 index 000000000..3a5db4c84 --- /dev/null +++ b/tests/fixtures/mtp_clause55/p0.txt @@ -0,0 +1,15 @@ +== ticket 'clause55 p0' 20261105-115131 hold=16min argv: --live --match --prompts 1 --prompt-base 0 --tokens 64 --k 4 --tau 0 +[ 169s] forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +[ 252s] prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, passes/token 0.625, stream 72 rows, 0 mismatch +[ 258s] match p0: 20 offline chains at this loop's own positions (stream 72 rows) +[ 258s] depth 1: live 90.00 % (18/20, reached) offline conditional 90.00 % (18/20) delta +0.00 pp [live joint 90.00 %, offline marginal 90.00 %] +[ 258s] depth 2: live 72.22 % (13/18, reached) offline conditional 72.22 % (13/18) delta +0.00 pp [live joint 65.00 %, offline marginal 65.00 %] +[ 258s] depth 3: live 53.85 % (7/13, reached) offline conditional 53.85 % (7/13) delta +0.00 pp [live joint 35.00 %, offline marginal 35.00 %] +[ 258s] depth 4: live 85.71 % (6/7, reached) offline conditional 85.71 % (6/7) delta +0.00 pp [live joint 30.00 %, offline marginal 35.00 %] +[ 258s] first verdict difference on the same token: stream position 13, depth 4: live reject, offline accept +[ 258s] == live vs offline at the SAME positions, per-depth acceptance CONDITIONAL on reaching the depth (the 2 pp bar): +[ 258s] depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp PASS +[ 258s] depth 2: live 72.22 % (13/18) offline 72.22 % (13/18) delta +0.00 pp PASS +[ 258s] depth 3: live 53.85 % (7/13) offline 53.85 % (7/13) delta +0.00 pp PASS +[ 258s] depth 4: live 85.71 % (6/7) offline 85.71 % (6/7) delta +0.00 pp PASS +== ticket 'clause55 p0' exited 0; log: /home/bcloud/wt/mtp-window/ticket-clause55-p0-20261105-115131.log diff --git a/tests/fixtures/mtp_clause55/p1-4.txt b/tests/fixtures/mtp_clause55/p1-4.txt new file mode 100644 index 000000000..1c9778ddf --- /dev/null +++ b/tests/fixtures/mtp_clause55/p1-4.txt @@ -0,0 +1,35 @@ +== ticket 'clause55 p1-4' 20261105-115551 hold=16min argv: --live --match --prompts 4 --prompt-base 1 --tokens 64 --k 4 --tau 0 +[ 66s] forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +[ 177s] prompt 1: TEXT IDENTICAL, 64 tokens, accepted 28/140, passes/token 1.109, stream 72 rows, 0 mismatch +[ 190s] match p1: 35 offline chains at this loop's own positions (stream 72 rows) +[ 190s] depth 1: live 60.00 % (21/35, reached) offline conditional 60.00 % (21/35) delta +0.00 pp [live joint 60.00 %, offline marginal 60.00 %] +[ 190s] depth 2: live 19.05 % (4/21, reached) offline conditional 19.05 % (4/21) delta +0.00 pp [live joint 11.43 %, offline marginal 22.86 %] +[ 191s] depth 3: live 75.00 % (3/4, reached) offline conditional 75.00 % (3/4) delta +0.00 pp [live joint 8.57 %, offline marginal 26.47 %] +[ 191s] depth 4: live 0.00 % (0/3, reached) offline conditional 0.00 % (0/3) delta +0.00 pp [live joint 0.00 %, offline marginal 8.82 %] +[ 191s] first verdict difference on the same token: stream position 9, depth 2: live reject, offline accept +[ 302s] prompt 2: TEXT IDENTICAL, 64 tokens, accepted 36/112, passes/token 0.875, stream 72 rows, 0 mismatch +[ 313s] match p2: 28 offline chains at this loop's own positions (stream 72 rows) +[ 313s] depth 1: live 60.71 % (17/28, reached) offline conditional 60.71 % (17/28) delta +0.00 pp [live joint 60.71 %, offline marginal 60.71 %] +[ 313s] depth 2: live 64.71 % (11/17, reached) offline conditional 64.71 % (11/17) delta +0.00 pp [live joint 39.29 %, offline marginal 42.86 %] +[ 313s] depth 3: live 45.45 % (5/11, reached) offline conditional 50.00 % (5/10) delta -4.55 pp > 2 pp [live joint 17.86 %, offline marginal 25.93 %] +[ 313s] depth 4: live 60.00 % (3/5, reached) offline conditional 60.00 % (3/5) delta +0.00 pp [live joint 10.71 %, offline marginal 19.23 %] +[ 313s] first verdict difference on the same token: stream position 13, depth 2: live reject, offline accept +[ 407s] prompt 3: TEXT IDENTICAL, 64 tokens, accepted 50/64, passes/token 0.500, stream 72 rows, 0 mismatch +[ 414s] match p3: 16 offline chains at this loop's own positions (stream 72 rows) +[ 414s] depth 1: live 87.50 % (14/16, reached) offline conditional 87.50 % (14/16) delta +0.00 pp [live joint 87.50 %, offline marginal 87.50 %] +[ 414s] depth 2: live 85.71 % (12/14, reached) offline conditional 85.71 % (12/14) delta +0.00 pp [live joint 75.00 %, offline marginal 75.00 %] +[ 414s] depth 3: live 100.00 % (12/12, reached) offline conditional 100.00 % (11/11) delta +0.00 pp [live joint 75.00 %, offline marginal 73.33 %] +[ 414s] depth 4: live 100.00 % (12/12, reached) offline conditional 100.00 % (11/11) delta +0.00 pp [live joint 75.00 %, offline marginal 80.00 %] +[ 414s] first verdict difference on the same token: stream position 9, depth 4: live reject, offline accept +[ 494s] prompt 4: TEXT IDENTICAL, 64 tokens, accepted 41/104, passes/token 0.812, stream 72 rows, 0 mismatch +[ 503s] match p4: 26 offline chains at this loop's own positions (stream 72 rows) +[ 503s] depth 1: live 76.92 % (20/26, reached) offline conditional 76.92 % (20/26) delta +0.00 pp [live joint 76.92 %, offline marginal 76.92 %] +[ 503s] depth 2: live 55.00 % (11/20, reached) offline conditional 52.63 % (10/19) delta +2.37 pp > 2 pp [live joint 42.31 %, offline marginal 40.00 %] +[ 503s] depth 3: live 54.55 % (6/11, reached) offline conditional 50.00 % (5/10) delta +4.55 pp > 2 pp [live joint 23.08 %, offline marginal 20.00 %] +[ 503s] depth 4: live 66.67 % (4/6, reached) offline conditional 60.00 % (3/5) delta +6.67 pp > 2 pp [live joint 15.38 %, offline marginal 12.50 %] +[ 503s] == live vs offline at the SAME positions, per-depth acceptance CONDITIONAL on reaching the depth (the 2 pp bar): +[ 503s] depth 1: live 68.57 % (72/105) offline 68.57 % (72/105) delta +0.00 pp PASS +[ 503s] depth 2: live 52.78 % (38/72) offline 52.11 % (37/71) delta +0.67 pp PASS +[ 503s] depth 3: live 68.42 % (26/38) offline 68.57 % (24/35) delta -0.15 pp PASS +[ 503s] depth 4: live 73.08 % (19/26) offline 70.83 % (17/24) delta +2.24 pp FAIL +== ticket 'clause55 p1-4' exited 0; log: /home/bcloud/wt/mtp-window/ticket-clause55-p1-4-20261105-115551.log diff --git a/tests/fixtures/mtp_clause55/p5-7.txt b/tests/fixtures/mtp_clause55/p5-7.txt new file mode 100644 index 000000000..3a4063e1e --- /dev/null +++ b/tests/fixtures/mtp_clause55/p5-7.txt @@ -0,0 +1,28 @@ +== ticket 'clause55 p5-7' 20261105-120713 hold=16min argv: --live --match --prompts 3 --prompt-base 5 --tokens 64 --k 4 --tau 0 +[ 49s] forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +[ 110s] prompt 5: TEXT IDENTICAL, 64 tokens, accepted 48/60, passes/token 0.484, stream 72 rows, 0 mismatch +[ 114s] match p5: 15 offline chains at this loop's own positions (stream 72 rows) +[ 114s] depth 1: live 86.67 % (13/15, reached) offline conditional 86.67 % (13/15) delta +0.00 pp [live joint 86.67 %, offline marginal 86.67 %] +[ 114s] depth 2: live 92.31 % (12/13, reached) offline conditional 92.31 % (12/13) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 114s] depth 3: live 100.00 % (12/12, reached) offline conditional 100.00 % (12/12) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 114s] depth 4: live 91.67 % (11/12, reached) offline conditional 91.67 % (11/12) delta +0.00 pp [live joint 73.33 %, offline marginal 80.00 %] +[ 114s] first verdict difference on the same token: stream position 9, depth 4: live reject, offline accept +[ 165s] prompt 6: TEXT IDENTICAL, 64 tokens, accepted 50/60, passes/token 0.469, stream 72 rows, 0 mismatch +[ 169s] match p6: 15 offline chains at this loop's own positions (stream 72 rows) +[ 169s] depth 1: live 93.33 % (14/15, reached) offline conditional 93.33 % (14/15) delta +0.00 pp [live joint 93.33 %, offline marginal 93.33 %] +[ 169s] depth 2: live 85.71 % (12/14, reached) offline conditional 85.71 % (12/14) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 169s] depth 3: live 100.00 % (12/12, reached) offline conditional 100.00 % (12/12) delta +0.00 pp [live joint 80.00 %, offline marginal 80.00 %] +[ 169s] depth 4: live 100.00 % (12/12, reached) offline conditional 100.00 % (11/11) delta +0.00 pp [live joint 80.00 %, offline marginal 78.57 %] +[ 231s] prompt 7: TEXT IDENTICAL, 64 tokens, accepted 41/104, passes/token 0.812, stream 72 rows, 0 mismatch +[ 237s] match p7: 26 offline chains at this loop's own positions (stream 72 rows) +[ 237s] depth 1: live 73.08 % (19/26, reached) offline conditional 73.08 % (19/26) delta +0.00 pp [live joint 73.08 %, offline marginal 73.08 %] +[ 237s] depth 2: live 57.89 % (11/19, reached) offline conditional 55.56 % (10/18) delta +2.34 pp > 2 pp [live joint 42.31 %, offline marginal 44.00 %] +[ 237s] depth 3: live 54.55 % (6/11, reached) offline conditional 50.00 % (5/10) delta +4.55 pp > 2 pp [live joint 23.08 %, offline marginal 28.00 %] +[ 237s] depth 4: live 83.33 % (5/6, reached) offline conditional 80.00 % (4/5) delta +3.33 pp > 2 pp [live joint 19.23 %, offline marginal 20.00 %] +[ 237s] first verdict difference on the same token: stream position 11, depth 2: live reject, offline accept +[ 237s] == live vs offline at the SAME positions, per-depth acceptance CONDITIONAL on reaching the depth (the 2 pp bar): +[ 237s] depth 1: live 82.14 % (46/56) offline 82.14 % (46/56) delta +0.00 pp PASS +[ 237s] depth 2: live 76.09 % (35/46) offline 75.56 % (34/45) delta +0.53 pp PASS +[ 237s] depth 3: live 85.71 % (30/35) offline 85.29 % (29/34) delta +0.42 pp PASS +[ 237s] depth 4: live 93.33 % (28/30) offline 92.86 % (26/28) delta +0.48 pp PASS +== ticket 'clause55 p5-7' exited 0; log: /home/bcloud/wt/mtp-window/ticket-clause55-p5-7-20261105-120713.log diff --git a/tests/mtp_depth_pool_test.py b/tests/mtp_depth_pool_test.py new file mode 100755 index 000000000..6ec01876f --- /dev/null +++ b/tests/mtp_depth_pool_test.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tools/mtp_depth_pool.py: the pooled per-depth live-vs-offline verdict, against fixture tickets. + +The tool exists because the brief judges "live acceptance equals the offline protocol within 2 pp at every +depth" on 8 prompts x 64 tokens POOLED, while a device ticket can only carry two or three prompts, and because +pooling `mtp_acceptance --live --match` logs by eye produced two wrong numbers in this lane. Both traps are +refusals in the tool, so they are tested here with fake tickets instead of only on the happy path: + +1. two tickets that agree pool; a ticket whose per-prompt rows disagree on a shared cell, or that does not add + up to its own run-level row (a short read), is refused rather than averaged; +2. the echoed summary a harness prints after its exit line is ignored, so a cell is never counted twice; +3. a depth with no denominator on either side is reported as unmeasured, never as a +0.00 pp pass. + +usage: tests/mtp_depth_pool_test.py (pure python, no model, no device) +""" +import os +import subprocess +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TOOL = os.path.join(ROOT, "tools", "mtp_depth_pool.py") + +# prompt 0: depth1 8/10 (offline 8/10), depth2 5/9 (5/9) +# prompt 1: depth1 7/10 (offline 6/10), depth2 4/9 (4/9) +TICKET_A = """forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +prompt 0: TEXT IDENTICAL, 64 tokens, accepted 6/16, passes/token 0.500, stream 72 rows, 0 mismatch + match p0: 4 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10) +depth 2: live 55.56 % (5/9, reached) offline conditional 55.56 % (5/9) +prompt 1: TEXT IDENTICAL, 64 tokens, accepted 5/16, passes/token 0.500, stream 72 rows, 0 mismatch + match p1: 4 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 70.00 % (7/10, reached) offline conditional 60.00 % (6/10) +depth 2: live 44.44 % (4/9, reached) offline conditional 44.44 % (4/9) +depth 1: live 75.00 % (15/20) offline 70.00 % (14/20) delta +5.00 pp FAIL +depth 2: live 50.00 % (9/18) offline 50.00 % (9/18) delta +0.00 pp PASS +== ticket exited rc=0 (hold reached) +prompt 0: TEXT IDENTICAL, 64 tokens, accepted 6/16, passes/token 0.500, stream 72 rows, 0 mismatch +depth 1: live 40.00 % (4/10, reached) offline conditional 40.00 % (4/10) +""" + +# prompt 2: depth1 9/10 (offline 7/10), depth2 3/9 (3/9) +TICKET_B = """prompt 2: TEXT IDENTICAL, 64 tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch + match p2: 4 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 90.00 % (9/10, reached) offline conditional 70.00 % (7/10) +depth 2: live 33.33 % (3/9, reached) offline conditional 33.33 % (3/9) +depth 1: live 90.00 % (9/10) offline 70.00 % (7/10) delta +20.00 pp FAIL +depth 2: live 33.33 % (3/9) offline 33.33 % (3/9) delta +0.00 pp PASS +""" + +# pooled over A+B: depth1 live 24/30 = 80.00 % vs offline 21/30 = 70.00 % -> +10.00 pp FAIL +# depth2 live 12/27 = 44.44 % vs offline 12/27 = 44.44 % -> +0.00 pp PASS +POOLED_DEPTH1 = "24/30 = 80.00%" +POOLED_DEPTH2 = "12/27 = 44.44%" + +fails = 0 + + +def check(cond, what): + global fails + print(f"{'PASS' if cond else 'FAIL'} {what}") + if not cond: + fails += 1 + + +def run(args, logs): + paths = [] + for name, body in logs: + p = os.path.join(TMP, name) + with open(p, "w") as fh: + fh.write(body) + paths.append(p) + r = subprocess.run([sys.executable, TOOL] + args + paths, capture_output=True, text=True) + return r.returncode, r.stdout, r.stderr + + +TMP = tempfile.mkdtemp(prefix="mtp-depth-pool-test.") + +# 1. two tickets that agree: the pool is their union, keyed by (prompt, depth), and the tail echo is ignored. +rc, out, err = run([], [("a.log", TICKET_A), ("b.log", TICKET_B)]) +check(rc == 0, "two agreeing tickets pool without a refusal") +check("pooled 6 scored cells = 3 prompts x 2 depths" in out, "3 prompts x 2 depths = 6 cells (the echoed tail is not counted)") +check(POOLED_DEPTH1 in out, f"depth 1 pools to {POOLED_DEPTH1}") +check(POOLED_DEPTH2 in out, f"depth 2 pools to {POOLED_DEPTH2}") +check("+10.00 pp" in out and "FAIL" in out, "depth 1 is 10.00 pp out, so the depth itself reads FAIL") +check("1 of 2 scored depths within 2 pp" in out, "the verdict line counts one pass of two scored depths") + +# 2. the same ticket twice is not double counting: identical cells are the same cells. +rc2, out2, _ = run([], [("a1.log", TICKET_A), ("a2.log", TICKET_A)]) +check(rc2 == 0 and "pooled 4 scored cells" in out2, "the same ticket twice pools to the same 4 cells, not 8") + +# 3. two tickets disagreeing on one shared cell -- prompt 1 depth 2, which both A and C score -- is refused. +# C is internally consistent (its own run-level row matches its rows), so only the cross-ticket check can catch it. +TICKET_C = """prompt 1: TEXT IDENTICAL, 64 tokens, accepted 5/16, passes/token 0.500, stream 72 rows, 0 mismatch +depth 1: live 70.00 % (7/10, reached) offline conditional 60.00 % (6/10) +depth 2: live 44.44 % (4/9, reached) offline conditional 11.11 % (1/9) +depth 1: live 70.00 % (7/10) offline 60.00 % (6/10) delta +10.00 pp FAIL +depth 2: live 44.44 % (4/9) offline 11.11 % (1/9) delta +33.33 pp FAIL +""" +rc3, _, err3 = run([], [("a.log", TICKET_A), ("c.log", TICKET_C)]) +check(rc3 == 1, f"a disagreement between tickets is refused (rc={rc3})") +check("disagrees" in err3 or "scored twice" in err3, f"the refusal says what disagrees: {err3.strip()[:120]}") + +# 4. a short read: the per-prompt rows no longer add up to the ticket's own run-level row. +short = TICKET_A.replace("depth 2: live 44.44 % (4/9, reached) offline conditional 44.44 % (4/9)\n", "") +rc4, _, err4 = run([], [("short.log", short)]) +check(rc4 == 1, f"a log missing a cell is refused (rc={rc4})") +check("read incompletely" in err4, f"the refusal names the short read: {err4.strip()[:120]}") + +# 5. a depth with no denominator is unmeasured, not a pass. +floor0 = """prompt 0: TEXT IDENTICAL, 64 tokens, accepted 1/16, passes/token 0.500, stream 72 rows, 0 mismatch +depth 1: live 100.00 % (1/1, reached) offline conditional 100.00 % (1/1) +depth 2: live 0.00 % (0/0, reached) offline conditional 0.00 % (0/0) +depth 1: live 100.00 % (1/1) offline 100.00 % (1/1) delta +0.00 pp PASS +depth 2: live 0.00 % (0/0) offline 0.00 % (0/0) delta +0.00 pp PASS +""" +rc5, out5, _ = run([], [("floor0.log", floor0)]) +check(rc5 == 0, "a depth with an empty denominator does not fail the run") +check("no denominator" in out5 and "1 of 1 scored depths" in out5, + "the empty depth is reported as unmeasured and excluded from the scored count") + +# 6. a depth the LOG reported as `not scored` is carried through as unmeasured, not dropped. +# +# The tool prints `depth 4: n/a (live 0/0 reached, offline 15/20) not scored` instead of a `%` row whenever one +# side has no denominator (npu/mtp/mtp_head.h:96), and at tau>0 the chain gate stops before depths 3-4. A pool +# that only kept the rows it could score dropped the depth entirely, so the table and the verdict line read as a +# complete pass over a table that had quietly lost a depth -- the phantom-depth read (#916) one layer up. +NOTSCORED = """prompt 0: TEXT IDENTICAL, 64 tokens, accepted 20/40, passes/token 0.500, stream 72 rows, 0 mismatch + per-depth p0: 1 18/20 2 13/20 3 2/20 4 0/20 + match p0: 20 offline chains at this loop's own positions (stream 72 rows) +depth 1: live 90.00 % (18/20, reached) offline conditional 90.00 % (18/20) +depth 2: live 65.00 % (13/20, reached) offline conditional 65.00 % (13/20) +depth 3: live 10.00 % (2/20, reached) offline conditional 10.00 % (2/20) +depth 4: n/a (live 0/0 reached, offline 15/20) not scored +depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp PASS +depth 2: live 65.00 % (13/20) offline 65.00 % (13/20) delta +0.00 pp PASS +depth 3: live 10.00 % (2/20) offline 10.00 % (2/20) delta +0.00 pp PASS +depth 4: n/a (live 0/0 reached, offline 15/20) not scored +""" +rc6, out6, _ = run([], [("notscored.log", NOTSCORED)]) +check(rc6 == 0, "a log with a not-scored depth still pools") +check("unmeasured -- no chain reached this depth" in out6, "the not-scored depth is printed as unmeasured") +check("4 unmeasured" in out6, "... and it is depth 4, the one the log named") +check("3 of 3 scored depths" in out6 and "1 of 4 depths unmeasured: 4" in out6, + f"the verdict says 3 of 4 depths, not a bare '3 of 3': {out6.strip().splitlines()[-1]}") + +# 7. a pooled prompt set with a hole says so, instead of printing the range it spans. +# +# This lane runs one or two prompts per ticket, so prompts 0,1,5,6,7 is a normal pool -- but printed as +# "prompts 0..7" it reads as the brief's whole window. +def ticket(p, tokens=64): + return (f"prompt {p}: TEXT IDENTICAL, {tokens} tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch\n" + f"depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10)\n" + f"depth 1: live 80.00 % (8/10) offline 80.00 % (8/10) delta +0.00 pp PASS\n") + + +rc7, out7, _ = run([], [(f"p{p}.log", ticket(p)) for p in (0, 1, 5)]) +check(rc7 == 0, "a gapped prompt set pools without a refusal (per-prompt tickets are the normal case)") +check("NOT a contiguous window: 2,3,4 missing of 0..5" in out7, + f"the header names the hole instead of spanning it: {out7.splitlines()[0]}") +check("64 tokens/prompt" in out7, "a uniform budget is stated in the header") +rc8, out8, _ = run([], [(f"q{p}.log", ticket(p)) for p in (0, 1, 2)]) +check("prompts 0..2" in out8 and "NOT a contiguous" not in out8, "a contiguous pool keeps the plain range form") + +# 8. the generation budget is part of the window: 8 prompts x 64 tokens is the brief's, so a pool that mixes +# budgets says so instead of inheriting the shape of whichever ticket it read first. +rc9, out9, _ = run([], [("t64.log", ticket(0)), ("t128.log", ticket(1, tokens=128))]) +check(rc9 == 0, "mixed budgets still pool (a longer prompt is a legitimate measurement)") +check("MIXED token budgets" in out9 and "64 on prompts 0" in out9 and "128 on prompts 1" in out9, + f"the mixed budget is named per prompt: {[l for l in out9.splitlines() if 'MIXED' in l]}") + +# 9. the same prompt at two lengths is a contradiction about the same cell, not a wider window. +rc10, _, err10 = run([], [("same64.log", ticket(3)), ("same32.log", ticket(3, tokens=32))]) +check(rc10 == 1 and "two lengths of the same prompt" in err10, + f"prompt 3 at 64 and at 32 tokens is refused: {err10.strip()[:110]}") + +# 10. a log whose per-prompt rows predate the `reached` column is a DIFFERENT quantity, and says so: its live +# rate is marginal, not conditional, so "the log was read incompletely" would send a reader to re-run a complete +# ticket. (2 of the 16 cell-bearing logs in this lane's own corpus are this shape.) +OLD_SHAPE = """prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, passes/token 0.625, stream 72 rows, 0 mismatch + match p0: 20 offline chains at this loop's own positions (stream 72 rows) + depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp + depth 2: live 65.00 % (13/20) offline 65.00 % (13/20) delta +0.00 pp + == live vs offline at the SAME positions (the 2 pp bar): + depth 1: live 90.00 % (18/20) offline 90.00 % (18/20) delta +0.00 pp PASS + depth 2: live 65.00 % (13/20) offline 65.00 % (13/20) delta +0.00 pp PASS +""" +rc11, _, err11 = run([], [("oldshape.log", OLD_SHAPE)]) +check(rc11 == 1, "a pre-`reached` log is refused rather than pooled as if it were the current shape") +check("pre-`reached` shape" in err11 and "read incompletely" not in err11, + f"the refusal names the real cause, not a short read: {err11.strip()[-120:]}") + +# 12. k is part of the window too: a k=6 ticket pooled with k=4 ones builds a six-depth table whose depths 5-6 +# exist on the wider prompts only, and `N of N scored depths` then reads as a pass over depths the rest never tried. +def ticket_k(p, k): + body = [f"prompt {p}: TEXT IDENTICAL, 64 tokens, accepted 4/16, passes/token 0.500, stream 72 rows, 0 mismatch"] + body += [f" depth {d}: live 50.00 % (10/20, reached) offline conditional 50.00 % (10/20)" for d in range(1, k + 1)] + body.append(" == live vs offline at the SAME positions (the 2 pp bar):") + body += [f" depth {d}: live 50.00 % (10/20) offline 50.00 % (10/20) delta +0.00 pp PASS" + for d in range(1, k + 1)] + return "\n".join(body) + "\n" + + +rc12, out12, _ = run([], [("k4.log", ticket_k(0, 4)), ("k6.log", ticket_k(1, 6))]) +check(rc12 == 0, "a pool of runs at different k still pools (the cells themselves are not in conflict)") +check("MIXED chain width (k)" in out12 and "6 on prompts 1" in out12 and "4 on prompts 0" in out12, + f"the differing chain width is named per prompt: {[l for l in out12.splitlines() if 'MIXED chain' in l]}") +check("the depths above 4 come from the wider prompts only" in out12, + "... and says what it means for the table (depths 5-6 are not the whole pool)") +rc13, out13, _ = run([], [("k4a.log", ticket_k(0, 4)), ("k4b.log", ticket_k(1, 4))]) +check("k=4" in out13 and "MIXED chain width" not in out13, "a uniform chain width is stated once in the header") + +# 14. the run identity (vocab, batch width, hidden source) is part of the window: 2.1 vs 3.19 tokens/pass on the +# hidden-source switch alone is more than the bar, so pooling across it would average two measurements. +READY_NORM = "forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm\n" +READY_RSD = "forward ready: vocab 248320, max_batch_rows 16, hidden source residual\n" +rc14, out14, _ = run([], [("norm.log", READY_NORM + ticket_k(0, 4)), ("rsd.log", READY_RSD + ticket_k(1, 4))]) +check(rc14 == 0, "a pool across hidden sources still pools (the cells themselves are not in conflict)") +check("MIXED run identity" in out14 and "post-final-norm" in out14 and "residual" in out14, + f"a differing hidden source is named rather than averaged: {[l for l in out14.splitlines() if 'MIXED run' in l]}") +rc15, out15, _ = run([], [("n1.log", READY_NORM + ticket_k(0, 4)), ("n2.log", READY_NORM + ticket_k(1, 4))]) +check("vocab 248320, max_batch_rows 16, hidden post-final-norm" in out15 and "MIXED run identity" not in out15, + "a uniform run identity is stated in the header") + +# 15. the recorded 8x64 window, from the three tickets shipped under tests/fixtures/mtp_clause55 (see its README). +# Without this the known answer is only checkable on the machine that holds the original `npu-hold` logs, i.e. the tool +# that produces the deliverable's published verdict could break it and CI would not notice. +FIXTURES = os.path.join(ROOT, "tests", "fixtures", "mtp_clause55") +CL = [os.path.join(FIXTURES, n) for n in ("p0.txt", "p1-4.txt", "p5-7.txt")] +# A fixture that is present on the machine but not COMMITTED is the worse failure mode: the test passes locally and the +# CI checkout has nothing to read. It happened with `.log` (`.gitignore:19`), so being tracked is checked here when the +# tree is a git checkout, and the files carry a suffix the ignore rules do not match. +if os.path.isdir(os.path.join(ROOT, ".git")) or os.path.exists(os.path.join(ROOT, ".git")): + listed = subprocess.run(["git", "-C", ROOT, "ls-files", "--"] + [os.path.relpath(p, ROOT) for p in CL], + capture_output=True, text=True) + tracked = len([l for l in listed.stdout.splitlines() if l.strip()]) + check(tracked == len(CL), f"the three fixtures are tracked by git ({tracked} of {len(CL)})") +if all(os.path.exists(p) for p in CL): + r16 = subprocess.run([sys.executable, TOOL, "--selftest"] + CL, capture_output=True, text=True) + check(r16.returncode == 0, f"the shipped fixtures reproduce the recorded 8x64 window (rc={r16.returncode})") + check("32 scored cells = 8 prompts x 4 depths" in r16.stdout, "the fixtures pool to the recorded 8 x 4 cells") + for want in ("136/181 = 75.14%", "86/136 = 63.24%", "63/86 = 73.26%", "53/63 = 84.13%"): + check(want in r16.stdout, f"fixture pool reports {want}") + check("4 of 4 scored depths within 2 pp" in r16.stdout, "all four depths are inside the bar on the fixtures") + check("selftest PASS: 4 depths reproduce the recorded 8x64 window" in r16.stdout, + "the tool's own known-answer check passes on them") +else: + check(False, f"tests/fixtures/mtp_clause55 is incomplete: {CL}") + +# 16. the tool's own refusals, and (no logs) nothing else: this is what runs on a box without the ticket corpus. +r = subprocess.run([sys.executable, TOOL, "--selftest"], capture_output=True, text=True) +check(r.returncode == 0 and "selftest PASS tail repeat" in r.stdout, "the tool's refusal self-test passes") + +print(f"\n{'all checks passed' if not fails else f'{fails} CHECK(S) FAILED'}") +sys.exit(1 if fails else 0) diff --git a/tools/copyright.py b/tools/copyright.py index bd0391afa..9f47afa08 100644 --- a/tools/copyright.py +++ b/tools/copyright.py @@ -52,6 +52,10 @@ # engine#315 evidence artefacts that cannot carry a comment: diffs must stay byte-applicable # and data files have no comment syntax. Matches the existing "patches/*" exemption. "*.patch", "*.csv", "*.cfg", + # engine#1146: the clause-55 acceptance tickets are DATA a parser reads byte-verbatim (like + # tests/golden/*/tokens.txt), so a notice comment would make the fixture no longer the log it claims + # to be; the directory README carries the notice instead + "tests/fixtures/mtp_clause55/*.txt", # os/v2 packages: dpkg control files, apt/dpkg list files and the console banner are data # with no comment syntax - the same exemption patches/* and *.cfg get "os/v2/debian/*/control", "os/v2/debian/*/conffiles", "*.list", diff --git a/tools/mtp_depth_pool.py b/tools/mtp_depth_pool.py new file mode 100755 index 000000000..329068b80 --- /dev/null +++ b/tools/mtp_depth_pool.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# mtp_depth_pool.py -- pool the live-vs-offline per-depth table over several `mtp_acceptance --live --match` +# runs, and say whether the brief's clause ("live acceptance equals the offline protocol within 2 pp at every +# depth") holds on the pooled window. +# +# Why pooling exists at all: the clause is judged on 8 prompts x 64 tokens POOLED, but one `--live` run costs +# ~90 s of array time plus a model + head load per process, so an 8-prompt window is usually two or three +# npu-hold tickets of <= 3 min each. Pooling those logs by eye is how this lane produced two wrong numbers, +# so the arithmetic lives here and the two traps are refusals rather than comments: +# 1. a ticket log REPEATS ITS OWN TAIL after the harness' exit line, so a cell can appear twice -- pooling +# that once inflated a 12-cell window to 20 cells; +# 2. a cell is keyed by (prompt, depth), and a second row for the same key with different counts is a +# disagreement between two tickets (or a format change), never an average to be taken. +# A ticket's own run-level row is the sum of its per-prompt rows, so it is checked too: a truncated log cannot +# then pass silently as a smaller window. +# 3. a depth the tool reported as "n/a ... not scored" is carried through as UNMEASURED instead of being +# dropped, and the window is stated as it was measured rather than as it was asked for: a pooled prompt set +# that is not contiguous says which prompts are missing, the generation budget is printed (`64 tokens/prompt` +# or a MIXED line naming each length per prompt), and the chain width is printed (`k=4`, or which prompts ran +# wider). All of it is one failure: a pool that only reports the cells it scored can read as a complete pass +# over a window it never covered -- at tau>0 the chain gate stops before depths 3-4 (npu/mtp/mtp_head.h:96), +# this lane's tickets are per-prompt slices, and a k=6 ticket pooled with k=4 ones adds depths 5-6 that only +# its own prompts ever attempted. +# +# tools/mtp_depth_pool.py ticket-a.log ticket-b.log ... # pooled table + the 2 pp verdict per depth +# tools/mtp_depth_pool.py --selftest # the refusals and the arithmetic, no logs needed +# tools/mtp_depth_pool.py --selftest # ... and the recorded 8x64 window, to the count +# +# Exit: 0 the pool was computed (individual depths may still read FAIL), 1 a refusal, 2 no input. +"""tools/mtp_depth_pool.py -- the per-depth live-vs-offline verdict over several mtp_acceptance tickets.""" +import os +import re +import sys + +BAR = 2.0 # the brief's bar, in percentage points + +# The tool prints the per-depth table in two shapes, and confusing them is what this file exists not to repeat: +# per PROMPT (keyed, poolable): +# depth 2: live 55.00 % (11/20, reached) offline conditional 52.63 % (10/19) delta +2.37 pp > 2 pp +# per RUN (this ticket's own pool over its prompts, used only as the read-completeness check): +# depth 2: live 52.78 % (38/72) offline 52.11 % (37/71) delta +0.67 pp PASS +DETAIL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+), reached\)\s+" + r"offline conditional\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)") +# The third shape: the tool refuses to score a depth one side never reached and says so on both the per-prompt and +# the run-level lines (npu/mtp/mtp_head.h:96 -- "0/0 is not agreement"): +# depth 4: n/a (live 0/0 reached, offline 15/20) not scored +NOTSCORED = re.compile(r"depth (\d+):\s+n/a \(live (\d+)/(\d+) reached, offline (\d+)/(\d+)\)") + +# The summary section, and with it the run-level rows, starts here in every log that has per-prompt rows: +# == live vs offline at the SAME positions (the 2 pp bar): +# Before it, a `depth N: live .. (n/d) offline .. (n/d) delta ..` row is a PER-PROMPT row in the pre-`reached` +# shape (2026-11-02 and earlier), whose live rate is the MARGINAL one rather than the rate conditional on reaching +# the depth -- a different quantity from the current column, so such a log cannot be pooled at all. +SUMMARY = "== live vs offline at the SAME positions" + +# The run's own identity line, printed once per process at init: +# forward ready: vocab 248320, max_batch_rows 16, hidden source post-final-norm +# All three change what a cell means -- a different vocabulary is a different model, a different batch width is a +# different sealed ELF family, and the hidden source moves acceptance by more than the bar (this lane has measured +# 2.1 vs 3.19 tokens/pass on that one switch alone). A pool that mixes them is not one measurement. +IDENTITY = re.compile(r"forward ready:\s+vocab (\d+),\s+max_batch_rows (\d+),\s+hidden source (\S+)") +RUNLEVEL = re.compile(r"depth (\d+):\s+live\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+" + r"offline\s+-?[\d.]+ %\s+\((\d+)/(\d+)\)\s+delta") +PROMPT = re.compile(r"(?:per-depth p|prompt )(\d+)\b") +# The live per-prompt header carries the generation budget, and it is the only shape that carries it correctly: +# prompt 0: TEXT IDENTICAL, 64 tokens, accepted 44/80, passes/token 0.625, stream 72 rows, 0 mismatch +# (the offline header's "N tokens" at the same position is the stream length, not the budget, so it is not read +# here.) The brief's window is 8 prompts x 64 tokens, so a pool mixing budgets is not one window. +PROMPT_TOKENS = re.compile(r"prompt (\d+):\s+(?:TEXT IDENTICAL|TEXT DIVERGED),\s+(\d+) tokens,") + +# The numbers this tool must reproduce when it is pointed at the recorded window (TMTPLOOP-STATE.md, +# 2026-11-05): live (num, den) and offline (num, den) per depth, over 8 prompts x 64 tokens. +RECORDED = {1: (136, 181, 136, 181), 2: (86, 136, 84, 134), 3: (63, 86, 60, 82), 4: (53, 63, 49, 59)} + + +def parse_ticket(path): + """Return ({prompt, depth: counts}, {depths not scored}, {prompt: tokens emitted}) for one ticket log. + + Raises SystemExit on a log it must not pool: a depth row before any prompt header (nothing to key it to), + a cell scored twice with different counts, or per-prompt rows that do not add up to the ticket's own + run-level row (a short read).""" + cells, runlevel, prompt, not_scored, budget = {}, {}, None, set(), {} + old_shape, summary, width, ident = 0, False, {}, None + for ln in open(path, errors="replace").read().splitlines(): + if "== ticket exited" in ln: # trap 1: everything after this is the harness echoing the summary + break + if SUMMARY in ln: + summary = True + continue + m = IDENTITY.search(ln) + if m: + ident = (int(m.group(1)), int(m.group(2)), m.group(3)) + continue + m = PROMPT.search(ln) + if m and "depth " not in ln: + prompt = int(m.group(1)) + t = PROMPT_TOKENS.search(ln) + if t: + budget[prompt] = int(t.group(2)) + continue + m = DETAIL.search(ln) + if m: + if prompt is None: + raise SystemExit(f"mtp_depth_pool: {path}: a depth row before any prompt line -- " + f"refusing to guess which prompt it belongs to") + key = (prompt, int(m.group(1))) + width[prompt] = max(width.get(prompt, 0), key[1]) # the chain width k this prompt was drafted at + got = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) + if key in cells and cells[key] != got: + raise SystemExit(f"mtp_depth_pool: {path}: prompt {key[0]} depth {key[1]} scored twice with " + f"different counts {cells[key]} then {got}") + cells.setdefault(key, got) + continue + m = NOTSCORED.search(ln) + if m: + not_scored.add(int(m.group(1))) + if prompt is not None: + width[prompt] = max(width.get(prompt, 0), int(m.group(1))) + continue + m = RUNLEVEL.search(ln) + if m: + if not summary: + old_shape += 1 + runlevel[int(m.group(1))] = (int(m.group(2)), int(m.group(3)), int(m.group(4)), int(m.group(5))) + # A log whose per-prompt rows are all in the pre-`reached` shape cannot be pooled: its live rate is marginal + # and its offline rate unconditional, so calling that "the log was read incompletely" would send the reader to + # re-run a ticket that is complete -- 2 of the 16 cell-bearing logs in this lane's corpus are that shape. + if runlevel and not cells and old_shape: + raise SystemExit(f"mtp_depth_pool: {path}: {old_shape} per-prompt row(s) in the pre-`reached` shape (live " + f"rate marginal, not conditional on reaching the depth) -- this log predates the column, " + f"so its cells are a different quantity; re-run it with a current build rather than " + f"pooling it") + # The ticket's own run-level row is the pool of its per-prompt rows, so a short read cannot pass as a + # smaller window: the two have to agree before this log is allowed into the pool at all. + for d, rl in runlevel.items(): + got = tuple(sum(v[i] for (_, dd), v in cells.items() if dd == d) for i in range(4)) + if got != rl: + raise SystemExit(f"mtp_depth_pool: {path}: depth {d} per-prompt rows pool to {got} but the ticket's " + f"own run-level row says {rl} -- the log was read incompletely, do not pool it") + if runlevel and len(runlevel) != len({d for _, d in cells}): + raise SystemExit(f"mtp_depth_pool: {path}: run-level row covers {sorted(runlevel)} but per-prompt rows " + f"cover {sorted({d for _, d in cells})}") + return cells, not_scored, budget, width, ident + + +def pool(paths): + """Union the tickets' cells, refusing any disagreement on a key. + + Returns (cells, unmeasured, sources); `unmeasured` holds the depths some ticket reported as `not scored` + that no ticket scored a cell for -- kept, so the pool cannot report a pass over a depth it never saw.""" + seen, unmeasured, sources = {}, set(), 0 + budget, width, idents = {}, {}, {} + for p in paths: + rows, not_scored, bud, wid, ident = parse_ticket(p) + unmeasured |= not_scored + for pr, wid_k in wid.items(): + width[pr] = max(width.get(pr, 0), wid_k) # a deeper second run of one prompt wins, as its cells do + if ident: + idents.setdefault(ident, []).append(p) + for pr, tk in bud.items(): + if pr in budget and budget[pr] != tk: + raise SystemExit(f"mtp_depth_pool: {p} ran prompt {pr} for {tk} tokens but an earlier ticket ran " + f"it for {budget[pr]} -- two lengths of the same prompt are not one window") + budget.setdefault(pr, tk) + if not rows: + print(f"mtp_depth_pool: {p}: no matched rows (was it run with --live --match?)", file=sys.stderr) + continue + sources += 1 + for k, v in rows.items(): + if k in seen and seen[k] != v: + raise SystemExit(f"mtp_depth_pool: {p} disagrees with an earlier ticket on prompt {k[0]} " + f"depth {k[1]}: {seen[k]} then {v}") + seen.setdefault(k, v) + return seen, unmeasured - {d for _, d in seen}, sources, budget, width, idents + + +def coverage(prompts): + """The pooled prompt set, saying out loud when it is a slice with holes rather than a contiguous window. + + A missing prompt is not a formatting detail: the brief's window is prompts 0..7, so a pool over 0,1,5,6,7 + printed as "prompts 0..7" reads as the whole window while five of eight prompts were measured.""" + lo, hi, have = prompts[0], prompts[-1], set(prompts) + missing = [p for p in range(lo, hi + 1) if p not in have] + if not missing: + return f"prompts {lo}..{hi}" + return (f"prompts {','.join(str(p) for p in prompts)} -- NOT a contiguous window: " + f"{','.join(str(p) for p in missing)} missing of {lo}..{hi}") + + +def token_coverage(budget, prompts): + """The generation budget the pool actually covers, per prompt, and an explicit note when it is not one value.""" + got = {p: budget[p] for p in prompts if p in budget} + if not got: + return "unknown tokens/prompt", None + vals = sorted(set(got.values())) + if len(vals) == 1: + return f"{vals[0]} tokens/prompt", None + groups = "; ".join(f"{v} on prompts {','.join(str(p) for p in prompts if got.get(p) == v)}" + for v in vals) + return "MIXED token budgets", f"MIXED token budgets: {groups} -- the pooled verdict is not one window" + + +def width_coverage(width, prompts): + """The chain width (k) the pool actually covers, and an explicit note when it is not one value. + + k is part of the window, not a detail of one ticket: a k=6 ticket pooled with k=4 ones builds a six-depth table + whose depths 5-6 exist on the k=6 prompts only, and `N of N scored depths` then reads as a full pass over depths + the other tickets never attempted.""" + got = {p: width[p] for p in prompts if p in width} + if not got: + return "k unknown", None + vals = sorted(set(got.values())) + if len(vals) == 1: + return f"k={vals[0]}", None + groups = "; ".join(f"{v} on prompts {','.join(str(p) for p in prompts if got.get(p) == v)}" for v in vals) + return "MIXED k", (f"MIXED chain width (k): {groups} -- the depths above {vals[0]} come from the wider prompts " + f"only, not from the whole pool") + + +def ident_coverage(idents): + """The run identity the pool covers (vocab, batch width, hidden source), and a note when it is not one run.""" + if not idents: + return "run identity unstated", None + if len(idents) == 1: + v, m, h = next(iter(idents)) + return f"vocab {v}, max_batch_rows {m}, hidden {h}", None + parts = "; ".join(f"vocab {v}/M{m}/{h} from {os.path.basename(ps[0])}" for (v, m, h), ps in idents.items()) + return "MIXED run identity", (f"MIXED run identity: {parts} -- a different model, batch width or hidden source " + f"is a different measurement, not one pool") + + +def table(seen, sources, unmeasured=(), budget=None, width=None, idents=None): + """Print the pooled per-depth table; return (inside, scored, totals per depth).""" + depths = sorted({d for _, d in seen}) + prompts = sorted({p for p, _ in seen}) + tokens_note, mixed = token_coverage(budget or {}, prompts) + width_note, mixed_width = width_coverage(width or {}, prompts) + ident_note, mixed_ident = ident_coverage(idents or {}) + print(f"pooled {len(seen)} scored cells = {len(prompts)} prompts x {len(depths)} depths " + f"({coverage(prompts)}, {tokens_note}, {width_note}, {ident_note}, {sources} ticket(s))") + for note in (mixed, mixed_width, mixed_ident): + if note: + print(note) + print(f"\n{'depth':>5} {'live':>18} {'offline (same positions)':>24} {'delta':>9} verdict") + inside = scored = 0 + totals = {} + for d in depths: + live_n = sum(v[0] for (_, dd), v in seen.items() if dd == d) + live_d = sum(v[1] for (_, dd), v in seen.items() if dd == d) + off_n = sum(v[2] for (_, dd), v in seen.items() if dd == d) + off_d = sum(v[3] for (_, dd), v in seen.items() if dd == d) + totals[d] = (live_n, live_d, off_n, off_d) + if not live_d or not off_d: + # 0/0 is not agreement: a depth no chain reached is unmeasured, and scoring it as +0.00 pp PASS + # is how a phantom depth was once reported as a passing one (engine#916). + print(f"{d:>5} {'no denominator':>18}") + continue + delta = 100.0 * live_n / live_d - 100.0 * off_n / off_d + ok = abs(delta) <= BAR + scored += 1 + inside += ok + print(f"{d:>5} {f'{live_n}/{live_d} = {100.0 * live_n / live_d:.2f}%':>18} " + f"{f'{off_n}/{off_d} = {100.0 * off_n / off_d:.2f}%':>24} {f'{delta:+.2f} pp':>9} " + f"{'PASS' if ok else 'FAIL'}") + for d in sorted(unmeasured): + # Named, not omitted: the depth was asked for and no chain in any ticket reached it, so it is + # unmeasured -- and a table that simply lacks the row reads as a window that shrank to fit. + print(f"{d:>5} unmeasured -- no chain reached this depth in any ticket; not scored") + tail = "" + if unmeasured: + tail = (f" ({len(unmeasured)} of {len(depths) + len(unmeasured)} depths unmeasured: " + f"{','.join(str(d) for d in sorted(unmeasured))})") + print(f"\n{inside} of {scored} scored depths within {BAR:g} pp at the same positions{tail}") + return inside, scored, totals + + +def refusal_selftest(): + """The refusals are why this tool exists, so they are tested and not only the happy path.""" + import os + import tempfile + + d2 = "depth 2: live 72.22 % (13/18, reached) offline conditional 72.22 % (13/18)\n" + cases = [ + # a cell scored twice with different counts (two tickets disagreeing, or a format change) + ("differing duplicate", "prompt 0: TEXT IDENTICAL\n" + d2 + + "depth 2: live 50.00 % (9/18, reached) offline conditional 50.00 % (9/18)\n", "scored twice"), + # a log whose per-prompt rows do not add up to the ticket's own run-level row (short read) + ("incomplete read", "prompt 0: TEXT IDENTICAL\n" + d2 + + "depth 2: live 50.00 % (9/18) offline 50.00 % (9/18) delta +0.00 pp PASS\n", + "read incompletely"), + # a depth row with no prompt header to key it to + ("unkeyed depth row", d2, "before any prompt line"), + ] + for name, body, want in cases: + with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as fh: + fh.write(body) + path = fh.name + try: + parse_ticket(path) + except SystemExit as e: + if want not in str(e): + print(f"selftest FAIL {name}: refused with {str(e)!r}, wanted {want!r}", file=sys.stderr) + return 1 + print(f"selftest PASS {name}: refused -- {str(e).split(': ', 1)[-1][:96]}") + else: + print(f"selftest FAIL {name}: accepted a log it must refuse", file=sys.stderr) + return 1 + finally: + os.unlink(path) + + # The tail trap: the same cells repeated after the exit line must not double the window. + d1 = "depth 1: live 80.00 % (8/10, reached) offline conditional 80.00 % (8/10)\n" + run = "depth 1: live 80.00 % (8/10) offline 80.00 % (8/10) delta +0.00 pp PASS\n" + tail = ("== ticket exited rc=0 (hold reached)\n" + "prompt 0: TEXT IDENTICAL\n" + d1 + + "depth 1: live 40.00 % (4/10, reached) offline conditional 40.00 % (4/10)\n") + with tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) as fh: + fh.write("prompt 0: TEXT IDENTICAL\n" + d1 + run + tail) + path = fh.name + try: + cells, _, _, _, _ = parse_ticket(path) + if cells != {(0, 1): (8, 10, 8, 10)}: + print(f"selftest FAIL tail repeat: parsed {cells}, wanted only the first row", file=sys.stderr) + return 1 + print("selftest PASS tail repeat: the echoed summary after the exit line is ignored") + finally: + os.unlink(path) + return 0 + + +def main(argv): + selftest = "--selftest" in argv + paths = [a for a in argv if a != "--selftest"] + if not paths: + if selftest: + return refusal_selftest() + print(__doc__) + return 2 + seen, unmeasured, sources, budget, width, idents = pool(paths) + if not seen: + print("mtp_depth_pool: nothing to pool", file=sys.stderr) + return 1 + _, _, totals = table(seen, sources, unmeasured, budget, width, idents) + if not selftest: + return 0 + bad = {d: (totals.get(d), RECORDED[d]) for d in RECORDED if totals.get(d) != RECORDED[d]} + if bad: + for d, (g, e) in bad.items(): + print(f"selftest FAIL depth {d}: pooled {g} != recorded {e}", file=sys.stderr) + return 1 + print(f"selftest PASS: {len(RECORDED)} depths reproduce the recorded 8x64 window to the count") + return refusal_selftest() + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))