From 2b4988335af229aad35c6a11d52d4ebf51b6588b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 14:01:18 -0400 Subject: [PATCH 1/2] npu/mtp: the forward adapter describes the deferral it uses, and its row accessor can no longer stop overriding Three defects in one small header, all of the same family as the #811 drift: * the file comment said the adapter snapshots the state per verify batch and restores the accepted prefix, because "the forward has no GDN deferral yet". The code has used the deferred path since #805 P6 (begin_verify stores each row's recurrence inputs, commit_verify_rows applies the accepted prefix); the snapshot design is what the deferral replaced, and the comment priced the loop at a state copy it does not take. * the members that comment left behind -- snap_ (unique_ptr) and snap_pos_ -- were dead: one declared and never touched, one assigned and never read. * last_hidden_rows() was the only non-override in the class, and it is the single signature the drafter's advance hangs on: drop the const and it silently stops overriding, reporting no rows at all, which is the failure mode that cost the live loop 1 token/pass for a day. Proof the guard works: with override, removing const from the accessor is now a compile error (`marked 'override', but does not override`), where before it was silent. mtp_acceptance builds clean, ctest -R mtp_draft 1/1, copyright --check clean. --- npu/mtp/forward_base.h | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/npu/mtp/forward_base.h b/npu/mtp/forward_base.h index 8df16a41f..4faa80766 100644 --- a/npu/mtp/forward_base.h +++ b/npu/mtp/forward_base.h @@ -16,10 +16,15 @@ // forward_base.h -- onebit::npu::mtp::BaseModel over the real Q4nxNpuForward, so the draft loop (#785, #798) // can drive the actual 27B instead of a toy. // -// The one non-obvious part is commit_verify(): the forward has no GDN deferral yet, so this adapter takes a -// state snapshot before the verify batch and restores the accepted prefix afterwards. That is the honest -// fallback documented in npu/mtp/draft.h -- correct, and slower than the deferred design (a snapshot per -// iteration instead of k rows of update inputs). When the engine grows deferral, only this file changes. +// The one non-obvious part is commit_verify(): a GDN layer's recurrence cannot be re-indexed, so the verify batch +// DEFERS it -- begin_verify() stores each row's recurrence inputs instead of applying them, and +// commit_verify_rows(accepted) applies exactly the accepted prefix and drops the rest (#805 P6). Nothing is +// snapshotted per iteration and no prefix restore is attempted (restore_state() refuses those on a GDN model); the +// forward's snapshot_state()/StateSnapshot API exists and this adapter deliberately does not use it. +// +// last_hidden_rows() is `override` for a reason: the loop's advance over the accepted rows hangs on that one +// signature, and a base whose method silently stops overriding reports no rows at all -- the failure mode that cost +// the live loop 1 token/pass for a day (#811). #pragma once #include "mtp/draft.h" @@ -27,7 +32,6 @@ #include "q4nx_forward.h" #include -#include namespace onebit::npu::mtp { @@ -46,7 +50,6 @@ class ForwardBase : public BaseModel { // recurrence inputs, so commit_verify() re-applies only the accepted rows. No state copy per // iteration and no prefix restore (restore_state() refuses those on a GDN model). fwd_.begin_verify((int)tokens.size(), start_pos); - snap_pos_ = start_pos; pending_ = (int)tokens.size(); last_tokens_ = tokens; return fwd_.step_batch_all(tokens, start_pos, logits_all); @@ -65,7 +68,7 @@ class ForwardBase : public BaseModel { return use_residual_ ? fwd_.last_residual() : fwd_.last_hidden(); } void use_residual(bool on) { use_residual_ = on; } - const std::vector& last_hidden_rows() const { return fwd_.last_hidden_rows(); } + const std::vector& last_hidden_rows() const override { return fwd_.last_hidden_rows(); } // The drafter needs the base embedding of the token it is drafting from; the forward owns that table. void embed_into(int token, std::vector& out) const { fwd_.embed_row(token, out); } @@ -75,9 +78,8 @@ class ForwardBase : public BaseModel { private: Q4nxNpuForward& fwd_; - std::unique_ptr snap_; std::vector last_tokens_; - int snap_pos_ = 0, pending_ = 0, NV_ = 248320; + int pending_ = 0, NV_ = 248320; bool use_residual_ = false; }; From 6f3c6d58ba7039439eb198310e9a0fca80d66d16 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Thu, 5 Nov 2026 15:18:28 -0400 Subject: [PATCH 2/2] npu: the adapter's comment says what a prefix restore DOES, not that it is refused Review nit on #1143, and correct: `restore_state()` (npu/forward/src/q4nx_forward.cpp:326-341) does not refuse a prefix. It refuses only a plane of the wrong size (`give_whole`) and a `tokens > in.tokens` extension; for `tokens < in.tokens` it restores the KV caches per position to `tokens` and then installs the GDN conv/recurrent planes WHOLE from the snapshot. So a prefix restore is accepted and leaves KV at the prefix with the recurrence still at the snapshot's last position -- exactly the mixed state the deferred commit exists to avoid, and exactly what the sentence was meant to warn about. The comment now says that, with the line reference, instead of claiming a guard that is not there. This PR exists because a comment asserted a design the code had replaced; a comment asserting a safety property that is not implemented is the same defect with the opposite sign, so it is fixed rather than merged with it. Comment only: `ninja mtp_acceptance` rc 0, `ctest -R mtp_draft` 1/1, `copyright --check` clean, no behaviour change (the adapter still never calls snapshot_state()/restore_state()). --- npu/mtp/forward_base.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/npu/mtp/forward_base.h b/npu/mtp/forward_base.h index 4faa80766..a94776994 100644 --- a/npu/mtp/forward_base.h +++ b/npu/mtp/forward_base.h @@ -19,7 +19,10 @@ // The one non-obvious part is commit_verify(): a GDN layer's recurrence cannot be re-indexed, so the verify batch // DEFERS it -- begin_verify() stores each row's recurrence inputs instead of applying them, and // commit_verify_rows(accepted) applies exactly the accepted prefix and drops the rest (#805 P6). Nothing is -// snapshotted per iteration and no prefix restore is attempted (restore_state() refuses those on a GDN model); the +// snapshotted per iteration and this adapter never restores: restore_state() ACCEPTS a prefix (`tokens < in.tokens`) +// and installs the GDN planes whole from the snapshot (`q4nx_forward.cpp:326-341`; its only refusals are a plane of +// the wrong size and a `tokens > in.tokens` extension), so a prefix restore here would leave KV at the prefix with +// the recurrence still at the snapshot's last position -- the mixed state this deferral exists to avoid. The // forward's snapshot_state()/StateSnapshot API exists and this adapter deliberately does not use it. // // last_hidden_rows() is `override` for a reason: the loop's advance over the accepted rows hangs on that one