diff --git a/docs/hw-gate.md b/docs/hw-gate.md index 389a6b4e1..3cf1d7ca2 100644 --- a/docs/hw-gate.md +++ b/docs/hw-gate.md @@ -144,11 +144,15 @@ The models (`MODEL_TABLE` in the script, files under `~/models`): | qwen3-0.6b | Qwen3-0.6B-Q4_K_M.gguf | small dense K-quant | 3 | yes | | zaya1-8b | zaya1-8b-Q4_K_M.gguf | ZAYA1 (16 experts, 1 active per token), K-quant | 2 | yes | | qwen3.8-27b | qwen38/UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf | gated delta net + attention hybrid, Unsloth dynamic mix | 2 | no | -| qwen3-coder-30b-a3b | Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf | qwen3moe: 128 experts, 8 active per token (MUL_MAT_ID with top-8 routing) | 2 | no | +| qwen3.6-35b-a3b | qwen36/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf | qwen35moe: 256 experts, 8 active per token (MUL_MAT_ID with top-8 routing; IQ4_XS / IQ3_S experts) | 2 | no | | qwen3-4b-iq4xs | Qwen3-4B-IQ4_XS.gguf | an IQ quant (IQ4_XS matmul and GET_ROWS) | 2 | no | | qwen3-0.6b-kq8 | Qwen3-0.6B-Q4_K_M.gguf, served with `-fa on -ctk q8_0` | a q8_0 K cache with an f16 V cache | 2 | yes | | qwen3-0.6b-kvq8 | Qwen3-0.6B-Q4_K_M.gguf, served with `-fa on -ctk q8_0 -ctv q8_0` | q8_0 K and V caches | 2 | yes | +The qwen3-coder-30b-a3b row (Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf, 128 experts) is out of the table until +#2494 closes (engine#2501): its multi-token prefill faults the GPU, so every gate run would reproduce that crash +trigger. The row stays in `MODEL_TABLE` as a comment; re-enable it when #2494 closes. + The last two rows give `llama-server` extra arguments (the fifth column of `MODEL_TABLE`). Their kat needle (about 4.7K tokens) and the 7.1K-token needle decode past 1,024 KV cells, where the add-on's split decode kernel (`fa_decode_q8_d128`) takes over from its prompt kernel; the short kat cases diff --git a/scripts/hw-gate.sh b/scripts/hw-gate.sh index 8831f3b5f..b17e46332 100755 --- a/scripts/hw-gate.sh +++ b/scripts/hw-gate.sh @@ -195,8 +195,8 @@ FLAKY_CASES=${HW_GATE_FLAKY:-"MUL_MAT(type_a=f32,type_b=f32,m=1,n=1,k=2048"} # name | file under $MODELS | kat --repeat | needle (1 = run tests/hrx_long_prompt.py on it) # | extra llama-server arguments -# The first three are K-quants (ZAYA1-8B routes 1 of 16 experts). The Qwen3-Coder row covers -# MUL_MAT_ID with top-8 of 128 experts, the IQ row the IQ4_XS matmul and GET_ROWS paths. The +# The first three are K-quants (ZAYA1-8B routes 1 of 16 experts). The Qwen3.6-35B-A3B row covers +# MUL_MAT_ID with top-8 of 256 experts (IQ4_XS / IQ3_S experts, qwen35moe), the IQ row the IQ4_XS matmul and GET_ROWS paths. The # needle runs on a small and a mid-size model. # The two -kq8/-kvq8 rows serve Qwen3-0.6B with a q8_0 K cache (V f16, `-ctk q8_0` alone) and with # q8_0 K and V. Their kat needle (~4.7K tokens) and the 7.1K-token needle decode past 1024 KV @@ -207,10 +207,13 @@ FLAKY_CASES=${HW_GATE_FLAKY:-"MUL_MAT(type_a=f32,type_b=f32,m=1,n=1,k=2048"} # 0..31 only (test-backend-ops has no FLASH_ATTN_EXT case with K and V of different types). # Rows with server arguments run kat, needle and placement, not kld: the KLD bases are stored per # model file and recorded with the default f16 cache. +# engine#2501: qwen3-coder-30b-a3b|Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf|2|0 -- re-enable when #2494 closes. Its +# multi-token prefill faults the GPU (MES REMOVE_QUEUE -> MODE2 reset), so every gate run would reproduce the crash +# trigger, and its GGUF was removed from ~/models. Its MoE coverage moved to the qwen3.6-35b-a3b row. MODEL_TABLE="qwen3-0.6b|Qwen3-0.6B-Q4_K_M.gguf|3|1 zaya1-8b|zaya1-8b-Q4_K_M.gguf|2|1 qwen3.8-27b|qwen38/UD-Q4_K_XL/Qwen3.8-27B-UD-Q4_K_XL.gguf|2|0 -qwen3-coder-30b-a3b|Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf|2|0 +qwen3.6-35b-a3b|qwen36/Qwen3.6-35B-A3B-UD-IQ4_XS.gguf|2|0 qwen3-4b-iq4xs|Qwen3-4B-IQ4_XS.gguf|2|0 qwen3-0.6b-kq8|Qwen3-0.6B-Q4_K_M.gguf|2|1|-fa on -ctk q8_0 qwen3-0.6b-kvq8|Qwen3-0.6B-Q4_K_M.gguf|2|1|-fa on -ctk q8_0 -ctv q8_0" diff --git a/tests/golden/hw-gate/kat-qwen3.6-35b-a3b.json b/tests/golden/hw-gate/kat-qwen3.6-35b-a3b.json new file mode 100644 index 000000000..ff831a030 --- /dev/null +++ b/tests/golden/hw-gate/kat-qwen3.6-35b-a3b.json @@ -0,0 +1,575 @@ +{ + "url": "http://127.0.0.1:18471", + "repeat": 2, + "cases": [ + { + "name": "capital", + "prompt_tokens": 24, + "text": "Paris", + "tokens": [ + { + "t": "Paris", + "lp": -0.2349565625190735, + "top": [ + -0.2349565625190735, + -1.565524935722351, + -9.084454536437988 + ] + }, + { + "t": "", + "lp": -8.118482219288126e-05, + "top": [ + -8.118482219288126e-05, + -10.251458168029785, + -10.423555374145508 + ] + } + ], + "expect_ok": true, + "secs": [ + 1.54, + 0.19 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "arithmetic", + "prompt_tokens": 28, + "text": "391", + "tokens": [ + { + "t": "3", + "lp": -5.996406980557367e-05, + "top": [ + -5.996406980557367e-05, + -10.654908180236816, + -11.369640350341797 + ] + }, + { + "t": "9", + "lp": -7.987054232216906e-06, + "top": [ + -7.987054232216906e-06, + -12.347261428833008, + -13.164413452148438 + ] + }, + { + "t": "1", + "lp": -7.74863383412594e-06, + "top": [ + -7.74863383412594e-06, + -12.461601257324219, + -13.464674949645996 + ] + }, + { + "t": "", + "lp": -6.55653229841846e-06, + "top": [ + -6.55653229841846e-06, + -12.099896430969238, + -14.456515312194824 + ] + } + ], + "expect_ok": true, + "secs": [ + 0.86, + 0.24 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "code", + "prompt_tokens": 32, + "text": "def add(a, b):\n return a + b", + "tokens": [ + { + "t": "def", + "lp": -0.3276808261871338, + "top": [ + -0.3276808261871338, + -1.2763617038726807, + -9.577491760253906 + ] + }, + { + "t": " add", + "lp": -1.406679530191468e-05, + "top": [ + -1.406679530191468e-05, + -12.509758949279785, + -13.316765785217285 + ] + }, + { + "t": "(a", + "lp": -4.172333774477011e-06, + "top": [ + -4.172333774477011e-06, + -13.785316467285156, + -13.79379653930664 + ] + }, + { + "t": ",", + "lp": -0.00031058842432685196, + "top": [ + -0.00031058842432685196, + -8.51480484008789, + -9.187366485595703 + ] + }, + { + "t": " b", + "lp": -3.695494797284482e-06, + "top": [ + -3.695494797284482e-06, + -13.253528594970703, + -14.42798900604248 + ] + }, + { + "t": "):", + "lp": -5.579150456469506e-05, + "top": [ + -5.579150456469506e-05, + -10.371253967285156, + -11.138111114501953 + ] + }, + { + "t": "\n", + "lp": -0.004962877836078405, + "top": [ + -0.004962877836078405, + -5.328954696655273, + -10.25861930847168 + ] + }, + { + "t": " ", + "lp": -0.0006701807142235339, + "top": [ + -0.0006701807142235339, + -8.02019214630127, + -8.62929630279541 + ] + }, + { + "t": " return", + "lp": -0.00018103569163940847, + "top": [ + -0.00018103569163940847, + -8.77412223815918, + -11.441242218017578 + ] + }, + { + "t": " a", + "lp": -1.0490472959645558e-05, + "top": [ + -1.0490472959645558e-05, + -12.160264015197754, + -13.089346885681152 + ] + }, + { + "t": " +", + "lp": -4.4108408474130556e-05, + "top": [ + -4.4108408474130556e-05, + -10.152076721191406, + -13.272445678710938 + ] + }, + { + "t": " b", + "lp": -5.364562093745917e-05, + "top": [ + -5.364562093745917e-05, + -10.317632675170898, + -11.95250129699707 + ] + }, + { + "t": "", + "lp": -8.213857654482126e-05, + "top": [ + -8.213857654482126e-05, + -9.577489852905273, + -12.729640007019043 + ] + } + ], + "expect_ok": true, + "secs": [ + 0.98, + 0.41 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "needle", + "prompt_tokens": 4739, + "text": "7341", + "tokens": [ + { + "t": "7", + "lp": -5.0307586207054555e-05, + "top": [ + -5.0307586207054555e-05, + -10.665112495422363, + -11.810765266418457 + ] + }, + { + "t": "3", + "lp": -1.3113030945532955e-06, + "top": [ + -1.3113030945532955e-06, + -14.363551139831543, + -14.653761863708496 + ] + }, + { + "t": "4", + "lp": -1.2040211004205048e-05, + "top": [ + -1.2040211004205048e-05, + -11.557461738586426, + -14.41179084777832 + ] + }, + { + "t": "1", + "lp": -5.9010340919485316e-05, + "top": [ + -5.9010340919485316e-05, + -9.897268295288086, + -12.545251846313477 + ] + }, + { + "t": "", + "lp": -5.865269122296013e-05, + "top": [ + -5.865269122296013e-05, + -10.127459526062012, + -11.20279598236084 + ] + } + ], + "expect_ok": true, + "secs": [ + 4.66, + 3.12 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "prefix-reuse", + "prompt_tokens": 4736, + "text": "green", + "tokens": [ + { + "t": "green", + "lp": -0.08852383494377136, + "top": [ + -0.08852383494377136, + -2.4777565002441406, + -7.940948486328125 + ] + }, + { + "t": "", + "lp": -4.0531240301788785e-06, + "top": [ + -4.0531240301788785e-06, + -13.362823486328125, + -13.489992141723633 + ] + } + ], + "expect_ok": true, + "secs": [ + 3.59, + 3.08 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "varied-180w", + "prompt_tokens": 250, + "text": "7341", + "tokens": [ + { + "t": "7", + "lp": -5.5910721130203456e-05, + "top": [ + -5.5910721130203456e-05, + -11.03247356414795, + -11.723461151123047 + ] + }, + { + "t": "3", + "lp": -1.7881409348774469e-06, + "top": [ + -1.7881409348774469e-06, + -14.088384628295898, + -15.32646656036377 + ] + }, + { + "t": "4", + "lp": -4.0531240301788785e-06, + "top": [ + -4.0531240301788785e-06, + -13.778643608093262, + -14.047412872314453 + ] + }, + { + "t": "1", + "lp": -4.136647839914076e-05, + "top": [ + -4.136647839914076e-05, + -10.807732582092285, + -11.156710624694824 + ] + }, + { + "t": "", + "lp": -5.746052920585498e-05, + "top": [ + -5.746052920585498e-05, + -10.574647903442383, + -10.607017517089844 + ] + } + ], + "expect_ok": true, + "secs": [ + 0.98, + 0.41 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "varied-720w", + "prompt_tokens": 946, + "text": "7341", + "tokens": [ + { + "t": "7", + "lp": -5.4122483561513945e-05, + "top": [ + -5.4122483561513945e-05, + -11.113892555236816, + -11.433526039123535 + ] + }, + { + "t": "3", + "lp": -9.536747711536009e-07, + "top": [ + -9.536747711536009e-07, + -14.358201026916504, + -15.800280570983887 + ] + }, + { + "t": "4", + "lp": -4.291543518775143e-06, + "top": [ + -4.291543518775143e-06, + -13.52664566040039, + -13.797008514404297 + ] + }, + { + "t": "1", + "lp": -3.528657180140726e-05, + "top": [ + -3.528657180140726e-05, + -10.597460746765137, + -12.074294090270996 + ] + }, + { + "t": "", + "lp": -4.983072358299978e-05, + "top": [ + -4.983072358299978e-05, + -10.572858810424805, + -10.867914199829102 + ] + } + ], + "expect_ok": true, + "secs": [ + 1.42, + 0.97 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "varied-1620w", + "prompt_tokens": 2136, + "text": "7341", + "tokens": [ + { + "t": "7", + "lp": -3.2187024771701545e-05, + "top": [ + -3.2187024771701545e-05, + -11.671987533569336, + -11.788705825805664 + ] + }, + { + "t": "3", + "lp": -8.344653679159819e-07, + "top": [ + -8.344653679159819e-07, + -14.268473625183105, + -16.22659683227539 + ] + }, + { + "t": "4", + "lp": -4.887592695013154e-06, + "top": [ + -4.887592695013154e-06, + -12.961409568786621, + -14.055787086486816 + ] + }, + { + "t": "1", + "lp": -3.695556370075792e-05, + "top": [ + -3.695556370075792e-05, + -10.454207420349121, + -12.221123695373535 + ] + }, + { + "t": "", + "lp": -2.4318991563632153e-05, + "top": [ + -2.4318991563632153e-05, + -11.23688793182373, + -11.594487190246582 + ] + } + ], + "expect_ok": true, + "secs": [ + 3.24, + 1.54 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + }, + { + "name": "varied-2520w", + "prompt_tokens": 3291, + "text": "7341", + "tokens": [ + { + "t": "7", + "lp": -3.659792128019035e-05, + "top": [ + -3.659792128019035e-05, + -11.30661392211914, + -11.878297805786133 + ] + }, + { + "t": "3", + "lp": -1.0728841743912199e-06, + "top": [ + -1.0728841743912199e-06, + -14.207064628601074, + -16.072790145874023 + ] + }, + { + "t": "4", + "lp": -5.1260126383567695e-06, + "top": [ + -5.1260126383567695e-06, + -12.76501750946045, + -14.047720909118652 + ] + }, + { + "t": "1", + "lp": -4.339312363299541e-05, + "top": [ + -4.339312363299541e-05, + -10.283246994018555, + -12.184653282165527 + ] + }, + { + "t": "", + "lp": -2.908749047492165e-05, + "top": [ + -2.908749047492165e-05, + -10.981531143188477, + -11.562541961669922 + ] + } + ], + "expect_ok": true, + "secs": [ + 3.23, + 2.16 + ], + "fails": [], + "warns": [], + "cache_path": { + "first_divergence": null + } + } + ], + "pass": true, + "time": "2026-11-15T05:16:18-0400" +} \ No newline at end of file diff --git a/tests/golden/hw-gate/manifest.json b/tests/golden/hw-gate/manifest.json index 0e01e291d..626282522 100644 --- a/tests/golden/hw-gate/manifest.json +++ b/tests/golden/hw-gate/manifest.json @@ -70,6 +70,29 @@ "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" } }, + "qwen3.6-35b-a3b": { + "args": "-c 512 --chunks 8 -b 512 -ub 512 --device HRX0 -ngl 99", + "base_bytes": 1013178324, + "base_sha256": "9a4b5a7b29debdaf2a47db9aeb8d1803aec8f14e33cd4cecf1548c05e0f2f2a1", + "limits": { + "max_kld_p99": 0.2, + "max_mean_kld": 0.01, + "min_same_top_p": 95.0 + }, + "model": { + "file": "Qwen3.6-35B-A3B-UD-IQ4_XS.gguf", + "sha256_first_64MiB": "fd7ca8d6957a0ab300d8f142c5e254de40bd64d4f0447f8be8303d026304891d", + "size": 17730509792 + }, + "recorded_at": { + "hrx-system": "7f4236924aadc069fb7425681525dddbd1535567", + "llama.cpp": "6b9fcacb7fbdff749060f25393577b8d7a5af5bc" + }, + "text": { + "file": "wiki.test.raw", + "sha256": "173c87a53759e0201f33e0ccf978e510c2042d7f2cb78229d9a50d79b9e7dd08" + } + }, "qwen3.8-27b": { "args": "-c 512 --chunks 8 -b 512 -ub 512 --device HRX0 -ngl 99", "base_bytes": 1013178324,