From 432caa734bf743268a8779c6d7aa749e84423e33 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 03:30:49 -0400 Subject: [PATCH 1/2] hw-gate: re-record the qwen3.8-27b row on the UD3.1 file the pin names (UNSLOTH-UD31) models/pins.json has pinned unsloth/Qwen3.8-27B-GGUF at the UD3.1 Qwen3.8-27B-UD-Q4_K_XL.gguf (2a5f3565, 17,659,407,264 B) since 2026-11-12; the gate's row was recorded on the earlier file (3f227079). The A/B adopted UD3.1 (PPL 5.9145 vs 5.9289, decode/pp512 equal), so the box's default path moves to it and the row follows, off the release pin by orchestrator decision. Recorded at llama.cpp 148e7f329 / hrx-system 7f423692 / gpu-kernels aa5f0e90b: kat 9/9, every answer text unchanged; check against it: KLD mean 0.000001, p99 0.00004. Co-Authored-By: Claude Opus 5.5 --- docs/hw-gate.md | 14 + tests/golden/hw-gate/kat-qwen3.8-27b.json | 446 +++++++++++----------- tests/golden/hw-gate/manifest.json | 10 +- 3 files changed, 242 insertions(+), 228 deletions(-) diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 3cf1d7ca2..f0dd9ed20 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -386,6 +386,20 @@ What the goldens record at the release pin: - On `varied-1620w`, the cached path answers differently from a fresh request at a near-tie: top-2 margin 0.19, a WARN. - ZAYA1-8B and Qwen3.8-27B answer every case. +The `qwen3.8-27b` row was re-recorded on 2026-11-16, off the release pin by orchestrator decision (UNSLOTH-UD31). +The reason is that the box moved to the file `models/pins.json` has named since 2026-11-12: unsloth's UD3.1 +`Qwen3.8-27B-UD-Q4_K_XL.gguf`, sha256 `2a5f3565…`, 17,659,407,264 B. The row was previously recorded on the +pre-UD3.1 file, `3f227079…`, 17,559,178,144 B. + +It was recorded at llama.cpp 148e7f329 / hrx-system 7f423692, with the private kernels at gpu-kernels aa5f0e90b. A +check run straight after it gives: + +- kat 9/9 with 0 warnings, and every answer's text identical to the old file's goldens (only the logprobs moved); +- KLD mean 0.000001, p99 0.00004, same top 100 %. + +The A/B measured the new file against the old base: mean KLD 0.0064, p99 0.065, same top 96.2 %, all inside the +limits. That is the size of the re-quantization, not a kernel change. + The MoE and IQ rows (`qwen3-coder-30b-a3b`, `qwen3-4b-iq4xs`) were recorded on 2026-10-20 at llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`; a submodule whose pin could not be read is recorded as `null` and named on stderr, never as the string "unknown"). Both answer every kat case. diff --git a/tests/golden/hw-gate/kat-qwen3.8-27b.json b/tests/golden/hw-gate/kat-qwen3.8-27b.json index 5979efeb0..44045555f 100644 --- a/tests/golden/hw-gate/kat-qwen3.8-27b.json +++ b/tests/golden/hw-gate/kat-qwen3.8-27b.json @@ -9,27 +9,27 @@ "tokens": [ { "t": "Paris", - "lp": -0.008619574829936028, + "lp": -0.0045010861940681934, "top": [ - -0.008619574829936028, - -4.80303430557251, - -8.56253433227539 + -0.0045010861940681934, + -5.631219387054443, + -7.783946990966797 ] }, { "t": "", - "lp": -2.47958396357717e-05, + "lp": -6.759395182598382e-05, "top": [ - -2.47958396357717e-05, - -10.848929405212402, - -12.940887451171875 + -6.759395182598382e-05, + -9.813271522521973, + -12.092218399047852 ] } ], "expect_ok": true, "secs": [ - 8.85, - 0.65 + 1.82, + 0.5 ], "fails": [], "warns": [], @@ -44,45 +44,45 @@ "tokens": [ { "t": "3", - "lp": -8.583106136939023e-06, + "lp": -1.9550514480215497e-05, "top": [ - -8.583106136939023e-06, - -12.15885066986084, - -13.918286323547363 + -1.9550514480215497e-05, + -11.830703735351562, + -12.30093765258789 ] }, { "t": "9", - "lp": -5.1260126383567695e-06, + "lp": -2.1457903130794875e-05, "top": [ - -5.1260126383567695e-06, - -12.925143241882324, - -12.968850135803223 + -2.1457903130794875e-05, + -11.08056640625, + -12.118644714355469 ] }, { "t": "1", - "lp": -5.1260126383567695e-06, + "lp": -2.4318991563632153e-05, "top": [ - -5.1260126383567695e-06, - -12.87148380279541, - -14.2367582321167 + -2.4318991563632153e-05, + -10.976825714111328, + -13.23958683013916 ] }, { "t": "", - "lp": -2.3841860752327193e-07, + "lp": -7.152559646783629e-07, "top": [ - -2.3841860752327193e-07, - -16.40816879272461, - -16.62249183654785 + -7.152559646783629e-07, + -14.653359413146973, + -16.28678321838379 ] } ], "expect_ok": true, "secs": [ - 3.43, - 0.79 + 1.78, + 0.64 ], "fails": [], "warns": [], @@ -97,171 +97,171 @@ "tokens": [ { "t": "```", - "lp": -0.14049369096755981, + "lp": -0.030796770006418228, "top": [ - -0.14049369096755981, - -2.0472586154937744, - -7.625899791717529 + -0.030796770006418228, + -3.5567424297332764, + -8.335551261901855 ] }, { "t": "python", - "lp": -4.589663149090484e-05, + "lp": -0.0002788097772281617, "top": [ - -4.589663149090484e-05, - -11.01229476928711, - -11.887155532836914 + -0.0002788097772281617, + -8.442216873168945, + -11.283349990844727 ] }, { "t": "\n", - "lp": -0.00019283962319605052, + "lp": -0.00025746561004780233, "top": [ - -0.00019283962319605052, - -9.068727493286133, - -9.908806800842285 + -0.00025746561004780233, + -8.721161842346191, + -9.742574691772461 ] }, { "t": "def", - "lp": -6.973986455705017e-05, + "lp": -9.024550672620535e-05, "top": [ - -6.973986455705017e-05, - -10.207085609436035, - -11.274020195007324 + -9.024550672620535e-05, + -9.925326347351074, + -11.22761058807373 ] }, { "t": " add", - "lp": -6.425587343983352e-05, + "lp": -7.844279025448486e-05, "top": [ - -6.425587343983352e-05, - -9.775490760803223, - -13.284026145935059 + -7.844279025448486e-05, + -9.628246307373047, + -12.879406929016113 ] }, { "t": "(a", - "lp": -1.2874686035502236e-05, + "lp": -2.1100266167195514e-05, "top": [ - -1.2874686035502236e-05, - -12.4428071975708, - -12.599102020263672 + -2.1100266167195514e-05, + -11.829000473022461, + -12.301395416259766 ] }, { "t": ",", - "lp": -0.00011528202594490722, + "lp": -0.00022813299437984824, "top": [ - -0.00011528202594490722, - -9.101000785827637, - -12.892406463623047 + -0.00022813299437984824, + -8.41391658782959, + -12.371569633483887 ] }, { "t": " b", - "lp": -2.861027041944908e-06, + "lp": -5.960482212685747e-06, "top": [ - -2.861027041944908e-06, - -13.072415351867676, - -14.742191314697266 + -5.960482212685747e-06, + -12.618535041809082, + -14.067562103271484 ] }, { "t": "):", - "lp": -7.39124880055897e-05, + "lp": -8.798032649792731e-05, "top": [ - -7.39124880055897e-05, - -9.805621147155762, - -11.615494728088379 + -8.798032649792731e-05, + -9.685115814208984, + -11.178622245788574 ] }, { "t": "\n", - "lp": -0.00010944011592073366, + "lp": -0.00013888847024645656, "top": [ - -0.00010944011592073366, - -9.52371597290039, - -11.391389846801758 + -0.00013888847024645656, + -9.362893104553223, + -11.080375671386719 ] }, { "t": " ", - "lp": -2.0384995877975598e-05, + "lp": -3.49289330188185e-05, "top": [ - -2.0384995877975598e-05, - -11.895397186279297, - -12.29019832611084 + -3.49289330188185e-05, + -11.268301963806152, + -11.630871772766113 ] }, { "t": " return", - "lp": -0.00010979778016917408, + "lp": -0.0005340213538147509, "top": [ - -0.00010979778016917408, - -9.508976936340332, - -10.516493797302246 + -0.0005340213538147509, + -7.619146347045898, + -10.411611557006836 ] }, { "t": " a", - "lp": -7.510213436034974e-06, + "lp": -1.2874686035502236e-05, "top": [ - -7.510213436034974e-06, - -12.792329788208008, - -13.08796215057373 + -1.2874686035502236e-05, + -12.408500671386719, + -12.660348892211914 ] }, { "t": " +", - "lp": -1.0371261851105373e-05, + "lp": -1.5378116586362012e-05, "top": [ - -1.0371261851105373e-05, - -12.07751750946045, - -13.287365913391113 + -1.5378116586362012e-05, + -11.63605785369873, + -13.009151458740234 ] }, { "t": " b", - "lp": -6.437322554120328e-06, + "lp": -1.2517053619376384e-05, "top": [ - -6.437322554120328e-06, - -12.494504928588867, - -13.80860424041748 + -1.2517053619376384e-05, + -11.863173484802246, + -13.159660339355469 ] }, { "t": "\n", - "lp": -2.789536301861517e-05, + "lp": -8.118482219288126e-05, "top": [ - -2.789536301861517e-05, - -10.912455558776855, - -12.416422843933105 + -8.118482219288126e-05, + -9.60977840423584, + -11.807193756103516 ] }, { "t": "```", - "lp": -0.00017316648154519498, + "lp": -0.00024303766258526593, "top": [ - -0.00017316648154519498, - -8.731668472290039, - -12.231843948364258 + -0.00024303766258526593, + -8.397574424743652, + -11.825374603271484 ] }, { "t": "", - "lp": -4.2439409298822284e-05, + "lp": -9.596808376954868e-05, "top": [ - -4.2439409298822284e-05, - -10.200047492980957, - -12.214926719665527 + -9.596808376954868e-05, + -9.412938117980957, + -11.189723014831543 ] } ], "expect_ok": true, "secs": [ - 4.53, - 2.01 + 3.13, + 1.9 ], "fails": [], "warns": [], @@ -276,54 +276,54 @@ "tokens": [ { "t": "7", - "lp": -8.643046749057248e-05, + "lp": -9.036472329171374e-05, "top": [ - -8.643046749057248e-05, - -10.2103271484375, - -11.323262214660645 + -9.036472329171374e-05, + -10.103190422058105, + -11.634066581726074 ] }, { "t": "3", - "lp": -3.755163197638467e-05, + "lp": -6.675942859146744e-05, "top": [ - -3.755163197638467e-05, - -10.255059242248535, - -13.910319328308105 + -6.675942859146744e-05, + -9.654502868652344, + -14.088135719299316 ] }, { "t": "4", - "lp": -3.409443888813257e-05, + "lp": -6.687864515697584e-05, "top": [ - -3.409443888813257e-05, - -10.332658767700195, - -15.014477729797363 + -6.687864515697584e-05, + -9.634984970092773, + -14.718888282775879 ] }, { "t": "1", - "lp": -1.8477610865375027e-05, + "lp": -3.7790057831443846e-05, "top": [ - -1.8477610865375027e-05, - -10.97034740447998, - -14.314371109008789 + -3.7790057831443846e-05, + -10.2210693359375, + -14.418076515197754 ] }, { "t": "", - "lp": -1.1920928955078125e-07, + "lp": 0.0, "top": [ - -1.1920928955078125e-07, - -16.47858238220215, - -16.860153198242188 + 0.0, + -17.556034088134766, + -17.94353485107422 ] } ], "expect_ok": true, "secs": [ - 14.98, - 9.38 + 10.64, + 9.15 ], "fails": [], "warns": [], @@ -338,27 +338,27 @@ "tokens": [ { "t": "green", - "lp": -0.14045651257038116, + "lp": -0.05574336275458336, "top": [ - -0.14045651257038116, - -2.5158228874206543, - -3.0021660327911377 + -0.05574336275458336, + -3.3888943195343018, + -3.8971760272979736 ] }, { "t": "", - "lp": -1.0728841743912199e-06, + "lp": -2.3841860752327193e-07, "top": [ - -1.0728841743912199e-06, - -14.096953392028809, - -15.354569435119629 + -2.3841860752327193e-07, + -15.703170776367188, + -15.777130126953125 ] } ], "expect_ok": true, "secs": [ - 9.57, - 9.04 + 9.22, + 8.84 ], "fails": [], "warns": [], @@ -373,54 +373,54 @@ "tokens": [ { "t": "7", - "lp": -0.0001587993756402284, + "lp": -0.00032096292125061154, "top": [ - -0.0001587993756402284, - -10.311230659484863, - -10.561487197875977 + -0.00032096292125061154, + -9.224732398986816, + -9.948224067687988 ] }, { "t": "3", - "lp": -6.997830496402457e-05, + "lp": -6.926299829501659e-05, "top": [ - -6.997830496402457e-05, - -9.646340370178223, - -12.687708854675293 + -6.926299829501659e-05, + -9.736895561218262, + -12.161120414733887 ] }, { "t": "4", - "lp": -7.319718133658171e-05, + "lp": -6.640177889494225e-05, "top": [ - -7.319718133658171e-05, - -9.556641578674316, - -14.35549545288086 + -6.640177889494225e-05, + -9.780189514160156, + -13.113313674926758 ] }, { "t": "1", - "lp": -7.963497773744166e-05, + "lp": -5.3049541747896e-05, "top": [ - -7.963497773744166e-05, - -9.477754592895508, - -13.621564865112305 + -5.3049541747896e-05, + -9.989910125732422, + -12.592750549316406 ] }, { "t": "", - "lp": -9.536747711536009e-07, + "lp": -2.98023678624304e-06, "top": [ - -9.536747711536009e-07, - -15.098958969116211, - -15.1939697265625 + -2.98023678624304e-06, + -13.812369346618652, + -14.076287269592285 ] } ], "expect_ok": true, "secs": [ - 2.28, - 1.14 + 1.46, + 1.0 ], "fails": [], "warns": [], @@ -435,54 +435,54 @@ "tokens": [ { "t": "7", - "lp": -0.000251742108957842, + "lp": -0.0005166673217900097, "top": [ - -0.000251742108957842, - -9.25206184387207, - -10.235177993774414 + -0.0005166673217900097, + -9.039740562438965, + -9.478696823120117 ] }, { "t": "3", - "lp": -3.361758354003541e-05, + "lp": -0.00010073692101286724, "top": [ - -3.361758354003541e-05, - -10.489279747009277, - -12.772956848144531 + -0.00010073692101286724, + -9.350265502929688, + -12.08779525756836 ] }, { "t": "4", - "lp": -2.0504208805505186e-05, + "lp": -8.941096166381612e-05, "top": [ - -2.0504208805505186e-05, - -10.93559455871582, - -14.107522964477539 + -8.941096166381612e-05, + -9.408042907714844, + -13.14515209197998 ] }, { "t": "1", - "lp": -1.9312092263135128e-05, + "lp": -6.389822374330834e-05, "top": [ - -1.9312092263135128e-05, - -10.951662063598633, - -14.512703895568848 + -6.389822374330834e-05, + -9.766753196716309, + -13.179192543029785 ] }, { "t": "", - "lp": -1.1920935776288388e-06, + "lp": -2.861027041944908e-06, "top": [ - -1.1920935776288388e-06, - -14.658820152282715, - -15.027756690979004 + -2.861027041944908e-06, + -14.022666931152344, + -14.147405624389648 ] } ], "expect_ok": true, "secs": [ - 5.47, - 2.62 + 2.78, + 2.32 ], "fails": [], "warns": [], @@ -497,54 +497,54 @@ "tokens": [ { "t": "7", - "lp": -0.00012744285049848258, + "lp": -0.0005169058567844331, "top": [ - -0.00012744285049848258, - -9.75546646118164, - -11.290661811828613 + -0.0005169058567844331, + -7.887544631958008, + -10.699525833129883 ] }, { "t": "3", - "lp": -3.540578472893685e-05, + "lp": -7.629685569554567e-05, "top": [ - -3.540578472893685e-05, - -10.359436988830566, - -13.440439224243164 + -7.629685569554567e-05, + -9.547843933105469, + -13.226019859313965 ] }, { "t": "4", - "lp": -3.194859891664237e-05, + "lp": -8.178091957233846e-05, "top": [ - -3.194859891664237e-05, - -10.422124862670898, - -14.346888542175293 + -8.178091957233846e-05, + -9.453811645507812, + -13.822009086608887 ] }, { "t": "1", - "lp": -1.895445711852517e-05, + "lp": -3.60018530045636e-05, "top": [ - -1.895445711852517e-05, - -10.944866180419922, - -14.66463851928711 + -3.60018530045636e-05, + -10.302864074707031, + -14.219688415527344 ] }, { "t": "", - "lp": -3.576279254957626e-07, + "lp": -2.3841860752327193e-07, "top": [ - -3.576279254957626e-07, - -16.103689193725586, - -16.471633911132812 + -2.3841860752327193e-07, + -16.281879425048828, + -16.380245208740234 ] } ], "expect_ok": true, "secs": [ - 7.24, - 4.85 + 5.01, + 4.45 ], "fails": [], "warns": [], @@ -559,54 +559,54 @@ "tokens": [ { "t": "7", - "lp": -0.0001394845894537866, + "lp": -0.0001452074502594769, "top": [ - -0.0001394845894537866, - -10.35625171661377, - -10.738471984863281 + -0.0001452074502594769, + -10.444262504577637, + -10.490718841552734 ] }, { "t": "3", - "lp": -4.1128048906102777e-05, + "lp": -5.161895387573168e-05, "top": [ - -4.1128048906102777e-05, - -10.207491874694824, - -13.38585090637207 + -5.161895387573168e-05, + -9.939674377441406, + -13.773594856262207 ] }, { "t": "4", - "lp": -4.2081763240275905e-05, + "lp": -6.47327397018671e-05, "top": [ - -4.2081763240275905e-05, - -10.13654899597168, - -14.440783500671387 + -6.47327397018671e-05, + -9.67629623413086, + -14.46218204498291 ] }, { "t": "1", - "lp": -2.3842141672503203e-05, + "lp": -2.8968277547392063e-05, "top": [ - -2.3842141672503203e-05, - -10.704211235046387, - -14.302668571472168 + -2.8968277547392063e-05, + -10.495759963989258, + -14.60205078125 ] }, { "t": "", - "lp": -2.3841860752327193e-07, + "lp": 0.0, "top": [ - -2.3841860752327193e-07, - -15.569476127624512, - -15.655967712402344 + 0.0, + -17.637432098388672, + -17.972341537475586 ] } ], "expect_ok": true, "secs": [ - 8.7, - 6.8 + 7.19, + 6.55 ], "fails": [], "warns": [], @@ -616,5 +616,5 @@ } ], "pass": true, - "time": "2026-10-25T10:48:48-0300" + "time": "2026-11-16T03:26:24-0400" } \ No newline at end of file diff --git a/tests/golden/hw-gate/manifest.json b/tests/golden/hw-gate/manifest.json index 626282522..673823d26 100644 --- a/tests/golden/hw-gate/manifest.json +++ b/tests/golden/hw-gate/manifest.json @@ -96,7 +96,7 @@ "qwen3.8-27b": { "args": "-c 512 --chunks 8 -b 512 -ub 512 --device HRX0 -ngl 99", "base_bytes": 1013178324, - "base_sha256": "a31cb56ca9a9f3361cdb9601af2bf219b8d1581761fbab47e91653558e9f32dd", + "base_sha256": "8b54bd01647a21134e6d87a799b26ff72a28cf7fc4c89fc59413803085903695", "limits": { "max_kld_p99": 0.2, "max_mean_kld": 0.01, @@ -104,12 +104,12 @@ }, "model": { "file": "Qwen3.8-27B-UD-Q4_K_XL.gguf", - "sha256_first_64MiB": "0bc73cb2357258a977f33f910a7d2b02c3c3ac7db98e4dcac6b58a8618b43827", - "size": 17559178144 + "sha256_first_64MiB": "54dc29992369431288d9aff9b0487116904a15e8d33ec4b22501fa03a5bcdc85", + "size": 17659407264 }, "recorded_at": { - "hrx-system": "82804197bcbe060b1371d6548337fdc842034cdd", - "llama.cpp": "4a58da9b087d23ac3596b35e58f3e69e29cd9f4b" + "hrx-system": "7f4236924aadc069fb7425681525dddbd1535567", + "llama.cpp": "148e7f329ff9b9dd3d7503ae4da5f53c6db604aa" }, "text": { "file": "wiki.test.raw", From e0067c172848441eb59a02429b218aa73661c17b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 04:00:27 -0400 Subject: [PATCH 2/2] hw-gate docs: name the limits the 27B re-quant delta is inside (#2646 review) Co-Authored-By: Claude Opus 5.5 --- docs/hw-gate.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/hw-gate.md b/docs/hw-gate.md index f0dd9ed20..87b2c5383 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -397,8 +397,9 @@ check run straight after it gives: - kat 9/9 with 0 warnings, and every answer's text identical to the old file's goldens (only the logprobs moved); - KLD mean 0.000001, p99 0.00004, same top 100 %. -The A/B measured the new file against the old base: mean KLD 0.0064, p99 0.065, same top 96.2 %, all inside the -limits. That is the size of the re-quantization, not a kernel change. +The A/B measured the new file against the old base: mean KLD 0.0064, p99 0.065, same top 96.2 %. That is inside +the row's own `limits` in `manifest.json` (`max_mean_kld` 0.01, `max_kld_p99` 0.2, `min_same_top_p` 95.0), and it +is the size of the re-quantization, not a kernel change. The MoE and IQ rows (`qwen3-coder-30b-a3b`, `qwen3-4b-iq4xs`) were recorded on 2026-10-20 at llama.cpp 00f2a68d / hrx-system 82804197 (`manifest.json` `recorded_at`; a submodule whose pin could not be read is