diff --git a/CMakeLists.txt b/CMakeLists.txt index de423b89..26ef8a9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -369,6 +369,7 @@ add_library(lse_model src/models/qwen3_5.cpp src/models/qwen3_5_common.cpp src/models/qwen3_5_moe.cpp + src/models/lfm3.cpp ) add_library(lse::model ALIAS lse_model) # Model kernels self-register from a TU nothing references. diff --git a/include/lse/model/config.hpp b/include/lse/model/config.hpp index f3cdecb1..398a3f8d 100644 --- a/include/lse/model/config.hpp +++ b/include/lse/model/config.hpp @@ -39,6 +39,16 @@ struct Config { std::int32_t attn_head_dim = 64; std::int32_t rope_dim = 64; float rope_theta = 10000.0f; + // RoPE profile for `sliding_attention` layers when it differs from full + // attention's (LFM3: theta 1e4 full-width vs 1e6 half-width). Zero/absent + // means sliding layers share the primary profile. + std::int32_t sliding_rope_dim = 0; + float sliding_rope_theta = 0.0f; + // The checkpoint's own layer schedule ("conv", "linear_attention", + // "full_attention", "sliding_attention"). Authoritative for mixer choice + // when populated; full_attention_interval is the fallback for configs that + // never declare it. + std::vector layer_types; // Gated DeltaNet std::int32_t gdn_qk_heads = 4; @@ -120,9 +130,19 @@ struct Config { } [[nodiscard]] bool is_attention_layer(std::int32_t layer_idx) const noexcept { + if (!layer_types.empty()) { + const std::string& t = layer_types[static_cast(layer_idx)]; + return t == "full_attention" || t == "sliding_attention"; + } return (layer_idx + 1) % full_attention_interval == 0; } + [[nodiscard]] bool is_sliding_attention(std::int32_t layer_idx) const noexcept { + return !layer_types.empty() && + layer_types[static_cast(layer_idx)] == + "sliding_attention"; + } + [[nodiscard]] bool is_global_attention(std::int32_t layer_idx) const noexcept { for (std::int32_t l : global_attention_layers) { if (l == layer_idx) return true; diff --git a/include/lse/tokenizer/tokenizer.hpp b/include/lse/tokenizer/tokenizer.hpp index d8a93a34..794c1f56 100644 --- a/include/lse/tokenizer/tokenizer.hpp +++ b/include/lse/tokenizer/tokenizer.hpp @@ -50,11 +50,17 @@ class Tokenizer { [[nodiscard]] bool valid() const noexcept { return handle_ != nullptr; } [[nodiscard]] void* raw() const noexcept { return handle_; } + // The id tokenizer_config.json declares as bos_token, 0 when none is + // declared. GPT-2-family checkpoints (LFM3) expect a leading BOS that the + // ByteLevel post-processor never injects — chat prompts prepend it to match + // HF/vLLM's add_special_tokens behavior; raw completions stay verbatim. + [[nodiscard]] std::uint32_t bos_token_id() const noexcept { return bos_id_; } DecodeStream stream(bool skip_special_tokens = true) const; private: void* handle_ = nullptr; + std::uint32_t bos_id_ = 0; }; // Incremental detokenization for streaming. Returns text only once a token diff --git a/src/kernels/sdpa.cpp b/src/kernels/sdpa.cpp index 58f1e269..1d0611f9 100644 --- a/src/kernels/sdpa.cpp +++ b/src/kernels/sdpa.cpp @@ -1212,7 +1212,7 @@ struct SdpaKernel final : KernelPrimitive { } else if (mask == 1) { if (auto g = e.when(j <= abs_i)) take(); } else { - if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) take(); + if (auto g = e.when(j <= abs_i && j + window > abs_i)) take(); } } @@ -1232,7 +1232,7 @@ struct SdpaKernel final : KernelPrimitive { } else if (mask == 1) { if (auto g = e.when(j <= abs_i)) apply(); } else { - if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) apply(); + if (auto g = e.when(j <= abs_i && j + window > abs_i)) apply(); } denom = denom.read() + w.read(); const auto vb = ((b * kvh + kh) * ts + j) * dv + d; @@ -1290,7 +1290,7 @@ struct SdpaKernel final : KernelPrimitive { } else if (mask == 3) { if (auto g = e.when(tree_visible(j))) take(); } else { - if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) take(); + if (auto g = e.when(j <= abs_i && j + window > abs_i)) take(); } } } @@ -1324,7 +1324,7 @@ struct SdpaKernel final : KernelPrimitive { } else if (mask == 3) { if (auto g = e.when(tree_visible(j))) apply(); } else { - if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) apply(); + if (auto g = e.when(j <= abs_i && j + window > abs_i)) apply(); } denom = denom.read() + w.read(); acc = math::fma(w.read(), kv_load(e, a.v, dim_major ? e.let(vb0 + jj) diff --git a/src/model/config.cpp b/src/model/config.cpp index 67869c76..c0c800c7 100644 --- a/src/model/config.cpp +++ b/src/model/config.cpp @@ -1,5 +1,6 @@ #include "lse/model/config.hpp" +#include #include #include #include @@ -71,6 +72,111 @@ Status read_storage_config(const nlohmann::json& root, Config& c) { return OkStatus(); } +// LFM3's text stack (Lfm3ForCausalLM, and the language model inside the +// Lfm3Omni wrapper), read off LiquidAI/LFM3-1.7B-MLX-4bit and +// LiquidAI/LFM3-Omni-2B-MLX-4bit and cross-checked against +// transformers/models/lfm3/modeling_lfm3.py. A conv/attention hybrid: +// `conv` layers are a gated depthwise width-7 convolution, attention layers +// carry per-head q/k norms, a per-head sigmoid output gate and a +// layer-type rotary profile. +Result from_lfm3_json(const nlohmann::json& root) { + const auto nested = root.find("text_config"); + const nlohmann::json& t = + nested != root.end() && nested->is_object() ? *nested : root; + Config c; + + LSE_RETURN_IF_ERROR(need(t, "vocab_size", c.vocab_size)); + LSE_RETURN_IF_ERROR(need(t, "hidden_size", c.hidden_size)); + LSE_RETURN_IF_ERROR(need(t, "num_hidden_layers", c.num_layers)); + LSE_RETURN_IF_ERROR(need(t, "norm_eps", c.rms_eps)); + LSE_RETURN_IF_ERROR(need(t, "num_attention_heads", c.attn_q_heads)); + LSE_RETURN_IF_ERROR(need(t, "num_key_value_heads", c.attn_kv_heads)); + LSE_RETURN_IF_ERROR(need(t, "head_dim", c.attn_head_dim)); + LSE_RETURN_IF_ERROR(need(t, "intermediate_size", c.mlp_intermediate)); + // The conv mixer's kernel width; the stack carries the kernel-1 tail. + LSE_RETURN_IF_ERROR(need(t, "conv_L_cache", c.gdn_conv_kernel)); + + // layer_types is the whole schedule: `conv` gets the short conv, + // `full_attention`/`sliding_attention` get attention. + if (!t.contains("layer_types") || !t["layer_types"].is_array()) { + return LSE_ERROR(kNotFound, "model config is missing 'layer_types'"); + } + c.layer_types = t["layer_types"].get>(); + if (static_cast(c.layer_types.size()) != c.num_layers) { + return LSE_ERROR(kInvalidArgument, "layer_types has ", + std::to_string(c.layer_types.size()), " entries but " + "num_hidden_layers is ", std::to_string(c.num_layers)); + } + for (const std::string& type : c.layer_types) { + if (type != "conv" && type != "full_attention" && + type != "sliding_attention") { + return LSE_ERROR(kUnimplemented, "layer_types names '", type, + "'; lfm3 mixers are conv, full_attention or " + "sliding_attention"); + } + } + + // Each attention kind owns a rotary profile: theta plus how much of the + // head rotates. Sliding defaults to full's when unlisted. + const auto ropes = t.find("rope_parameters"); + if (ropes == t.end() || !ropes->is_object()) { + return LSE_ERROR(kNotFound, "model config is missing 'rope_parameters'"); + } + auto read_rope = [&c](const nlohmann::json& r, float& theta, + std::int32_t& dim) -> Status { + LSE_RETURN_IF_ERROR(need(r, "rope_theta", theta)); + float partial = 1.0f; + read(r, "partial_rotary_factor", partial); + const auto rotary = static_cast( + static_cast(c.attn_head_dim) * partial + 0.5f); + if (rotary <= 0 || rotary % 2 != 0 || rotary > c.attn_head_dim) { + return LSE_ERROR(kInvalidArgument, "partial_rotary_factor ", + std::to_string(partial), " of head_dim ", + std::to_string(c.attn_head_dim), " gives a rope width " + "of ", std::to_string(rotary), + ", which must be even and positive"); + } + dim = rotary; + return OkStatus(); + }; + if (!ropes->contains("full_attention")) { + return LSE_ERROR(kNotFound, + "rope_parameters is missing 'full_attention'"); + } + LSE_RETURN_IF_ERROR( + read_rope((*ropes)["full_attention"], c.rope_theta, c.rope_dim)); + c.sliding_rope_theta = c.rope_theta; + c.sliding_rope_dim = c.rope_dim; + if (ropes->contains("sliding_attention") && + (*ropes)["sliding_attention"].is_object()) { + LSE_RETURN_IF_ERROR(read_rope((*ropes)["sliding_attention"], + c.sliding_rope_theta, c.sliding_rope_dim)); + } + + read(t, "sliding_window", c.sliding_window); + + // No delta-net layers, no experts: the non-attention mixer is the conv. + c.gdn_qk_heads = 0; + c.gdn_v_heads = 0; + c.gdn_head_dim = 0; + c.num_experts = 0; + c.num_active_experts = 0; + c.num_shared_experts = 0; + c.expert_intermediate = 0; + c.mtp_layers = 0; + // layer_types is authoritative; the interval rule is never consulted. + c.full_attention_interval = 1; + c.global_attention_layers.clear(); + + read(root, "tie_word_embeddings", c.tie_word_embeddings); + read(t, "tie_word_embeddings", c.tie_word_embeddings); + LSE_RETURN_IF_ERROR(read_storage_config(root, c)); + read(t, "max_position_embeddings", c.train_seq_len); + + LSE_RETURN_IF_ERROR(c.validate()); + return c; +} + // Qwen3.5 (dense and MoE), read off Qwen/Qwen3.5-0.8B, Qwen/Qwen3.5-4B and // Qwen/Qwen3.5-35B-A3B and cross-checked against // transformers/models/qwen3_5/configuration_qwen3_5.py. Both are gated @@ -195,7 +301,8 @@ Result from_hf_json(const nlohmann::json& root) { "num_hidden_layers is ", std::to_string(c.num_layers)); } for (std::size_t i = 0; i < types.size(); ++i) { - const bool attends = types[i] == "full_attention"; + const bool attends = types[i] == "full_attention" || + types[i] == "sliding_attention"; if (attends != c.is_attention_layer(static_cast(i))) { return LSE_ERROR(kUnimplemented, "layer_types says layer ", std::to_string(i), " is '", types[i], @@ -204,6 +311,7 @@ Result from_hf_json(const nlohmann::json& root) { " says otherwise"); } } + c.layer_types = types; } LSE_RETURN_IF_ERROR(c.validate()); @@ -229,6 +337,29 @@ Result Config::from_json_string(const std::string& text) { LSE_ASSIGN_OR(quant::GroupAffineMap quantization, quant::GroupAffineMap::from_config_json(text)); + // LFM3 (and Lfm3Omni's text stack) parses differently enough to own a + // branch: its config is HF-shaped but flat-or-nested depending on whether + // the checkpoint is the wrapper, and its rope_parameters are keyed per + // layer type rather than global. + std::string model_type; + if (j.contains("model_type") && j["model_type"].is_string()) + model_type = j["model_type"].get(); + if (model_type.empty() && j.contains("text_config") && + j["text_config"].is_object() && + j["text_config"].contains("model_type") && + j["text_config"]["model_type"].is_string()) + model_type = j["text_config"]["model_type"].get(); + std::string arch0; + if (j.contains("architectures") && j["architectures"].is_array() && + !j["architectures"].empty() && j["architectures"][0].is_string()) + arch0 = j["architectures"][0].get(); + if (model_type.rfind("lfm", 0) == 0 || arch0.rfind("Lfm", 0) == 0) { + LSE_ASSIGN_OR(Config lfm, from_lfm3_json(j)); + lfm.quantization = std::move(quantization); + lfm.sampling_defaults = sampling; + return lfm; + } + if (j.contains("text_config") && j["text_config"].is_object()) { LSE_ASSIGN_OR(Config hf, from_hf_json(j)); hf.quantization = std::move(quantization); @@ -251,6 +382,9 @@ Result Config::from_json_string(const std::string& text) { read(j, "attn_head_dim", c.attn_head_dim); read(j, "rope_dim", c.rope_dim); read(j, "rope_theta", c.rope_theta); + read(j, "sliding_rope_dim", c.sliding_rope_dim); + read(j, "sliding_rope_theta", c.sliding_rope_theta); + read(j, "layer_types", c.layer_types); read(j, "gdn_qk_heads", c.gdn_qk_heads); read(j, "gdn_v_heads", c.gdn_v_heads); @@ -321,6 +455,9 @@ std::string Config::to_json() const { j["attn_head_dim"] = attn_head_dim; j["rope_dim"] = rope_dim; j["rope_theta"] = rope_theta; + j["sliding_rope_dim"] = sliding_rope_dim; + j["sliding_rope_theta"] = sliding_rope_theta; + j["layer_types"] = layer_types; j["gdn_qk_heads"] = gdn_qk_heads; j["gdn_v_heads"] = gdn_v_heads; j["gdn_head_dim"] = gdn_head_dim; @@ -356,7 +493,7 @@ Status Config::validate() const { return LSE_ERROR(kInvalidArgument, "vocab_size, hidden_size and num_layers " "must all be positive"); } - if (full_attention_interval <= 0) { + if (full_attention_interval <= 0 && layer_types.empty()) { return LSE_ERROR(kInvalidArgument, "full_attention_interval must be positive"); } if (attn_q_heads % attn_kv_heads != 0) { @@ -364,8 +501,15 @@ Status Config::validate() const { std::to_string(attn_q_heads), ") to be a multiple of " "attn_kv_heads (", std::to_string(attn_kv_heads), ")"); } - if (gdn_qk_heads <= 0 || gdn_v_heads <= 0 || - (gdn_qk_heads % gdn_v_heads != 0 && gdn_v_heads % gdn_qk_heads != 0)) { + // Only checked when the stack actually runs delta-net layers — either + // every non-attention layer (layer_types unset, lemonseed's layout) or the + // ones the schedule names. LFM3's conv layers carry no recurrent state. + const bool has_gdn = layer_types.empty() || + std::find(layer_types.begin(), layer_types.end(), + "linear_attention") != layer_types.end(); + if (has_gdn && + (gdn_qk_heads <= 0 || gdn_v_heads <= 0 || + (gdn_qk_heads % gdn_v_heads != 0 && gdn_v_heads % gdn_qk_heads != 0))) { // Either direction: lemonseed's layer runs v at the key width and reads // gdn_v_heads as a divisor of it, Qwen shares 16 key heads across 32 value // heads. Both are GQA over the same state, counted from opposite ends. diff --git a/src/model/hybrid_lm.cpp b/src/model/hybrid_lm.cpp index 2116b3c3..80ec2ce0 100644 --- a/src/model/hybrid_lm.cpp +++ b/src/model/hybrid_lm.cpp @@ -1085,7 +1085,7 @@ Result HybridLM::hidden(const Array& tokens, const graph::ScopedMember on(shards > 1 ? i % shards : member_for_layer(layer)); - if (!st.gdn_state.valid()) { + if (vh_m > 0 && vd > 0 && !st.gdn_state.valid()) { LSE_ASSIGN_OR(st.gdn_state, mutable_zeros(Shape{batch, vh_m, vd, vd})); } if (tail <= 0) continue; @@ -1525,22 +1525,31 @@ Result HybridLM::hidden(const Array& tokens, if (t_live < t_now) { const auto heads = static_cast(config_.gdn_v_heads); const auto tail = static_cast(config_.gdn_conv_kernel) - 1; - if (heads <= 0 || tail <= 0 || t_live < tail) + if (heads <= 0 && tail <= 0) return LSE_ERROR(kInvalidArgument, "a padded pass needs at least ", std::to_string(tail), " real tokens and GDN value heads"); - std::vector m(static_cast(t_now * heads), 0.0f); - std::vector u(m.size(), 1.0f); - for (std::int64_t t = 0; t < t_live; ++t) - for (std::int64_t h = 0; h < heads; ++h) { - m[static_cast(t * heads + h)] = 1.0f; - u[static_cast(t * heads + h)] = 0.0f; - } - std::vector rows_of_tail(static_cast(tail)); - for (std::int64_t i = 0; i < tail; ++i) - rows_of_tail[static_cast(i)] = static_cast(t_live - tail + i); - LSE_ASSIGN_OR(mask, device_values(Shape{1, t_now, heads}, m)); - LSE_ASSIGN_OR(unmask, device_values(Shape{1, t_now, heads}, u)); - LSE_ASSIGN_OR(tail_rows, device_values(Shape{tail}, rows_of_tail)); + if (t_live < tail) + return LSE_ERROR(kInvalidArgument, "a padded pass needs at least ", + std::to_string(tail), " real tokens"); + // Per-head write suppression is a GDN-only need; the tail rows are what + // every conv-tail state (GDN or LFM-style) gathers on a padded pass. + if (heads > 0) { + std::vector m(static_cast(t_now * heads), 0.0f); + std::vector u(m.size(), 1.0f); + for (std::int64_t t = 0; t < t_live; ++t) + for (std::int64_t h = 0; h < heads; ++h) { + m[static_cast(t * heads + h)] = 1.0f; + u[static_cast(t * heads + h)] = 0.0f; + } + LSE_ASSIGN_OR(mask, device_values(Shape{1, t_now, heads}, m)); + LSE_ASSIGN_OR(unmask, device_values(Shape{1, t_now, heads}, u)); + } + if (tail > 0) { + std::vector rows_of_tail(static_cast(tail)); + for (std::int64_t i = 0; i < tail; ++i) + rows_of_tail[static_cast(i)] = static_cast(t_live - tail + i); + LSE_ASSIGN_OR(tail_rows, device_values(Shape{tail}, rows_of_tail)); + } } for (MixerState& st : *states) { st.pad_mask = mask; diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp new file mode 100644 index 00000000..81ac9ce2 --- /dev/null +++ b/src/models/lfm3.cpp @@ -0,0 +1,377 @@ +// LFM3: Liquid AI's conv/attention hybrid, read off +// LiquidAI/LFM3-1.7B-MLX-4bit and LiquidAI/LFM3-Omni-2B-MLX-4bit and +// cross-checked against transformers/models/lfm3/modeling_lfm3.py. +// +// A "conv" layer runs a gated depthwise width-7 convolution: in_proj emits +// B|C|x, the stream convolved is B*x, the result is gated by C and read out +// through out_proj. An attention layer is llama-style with per-head q/k +// RMSNorm and a per-head sigmoid output gate — gate_proj is [q_heads, +// hidden], expanded to per-channel rows at load because merging heads gives +// each of a head's channels the same gate value either way. Both attention +// kinds carry their own rotary profile (theta and rotary width), and every +// block is a sandwich: post_operator_norm and post_ffn_norm sit inside the +// residual adds, so they live inside the mixer and FFN impls. +// +// Text only: Omni checkpoints carry audio, vision and depth towers beside +// this stack; the spec's refusal list names them and reports the cost. + +#include +#include +#include +#include + +#include "lse/core/dtype.hpp" +#include "lse/graph/graph.hpp" +#include "lse/graph/ops.hpp" +#include "lse/model/config.hpp" +#include "lse/model/hybrid_lm.hpp" +#include "lse/model/layer.hpp" +#include "lse/model/qwen3_5_common.hpp" +#include "lse/model/registry.hpp" +#include "lse/model/weights.hpp" +#include "lse/ops/activation.hpp" +#include "lse/ops/attention.hpp" +#include "lse/ops/norm.hpp" +#include "lse/ops/rope.hpp" + +namespace lse::model { + +namespace { + +constexpr std::string_view kBlockPrefix = "language_model.model.layers"; + +// Same convention as qwen3_5: the engine's rotary pairs adjacent channels +// (2j, 2j+1) while HF's rotate_half pairs d with d + rope_dim/2, so the +// projection and head-norm rows are permuted once at load. The dot products +// attention reads are unchanged because q and k share the permutation. +std::int64_t rope_source(std::int64_t d, std::int64_t rope_dim) { + if (d >= rope_dim) return d; + const std::int64_t j = d / 2; + return d % 2 == 0 ? j : j + rope_dim / 2; +} + +class Lfm3Attention final : public IMixer { + public: + static constexpr std::string_view kName = "lfm3.attention"; + std::string_view name() const noexcept override { return kName; } + + Status load(WeightBinder& b, std::string_view prefix, + const LayerContext& ctx) override { + const Config& c = *ctx.config; + const std::string p = std::string(prefix) + ".self_attn"; + const auto hidden = static_cast(c.hidden_size); + const auto qh = static_cast(c.attn_q_heads); + const auto kvh = static_cast(c.attn_kv_heads); + const auto hd = static_cast(c.attn_head_dim); + const bool sliding = c.is_sliding_attention(ctx.layer_index); + const auto rd = static_cast( + sliding ? c.sliding_rope_dim : c.rope_dim); + const float theta = sliding ? c.sliding_rope_theta : c.rope_theta; + + std::vector q_rows; + q_rows.reserve(static_cast(qh * hd)); + for (std::int64_t h = 0; h < qh; ++h) + for (std::int64_t d = 0; d < hd; ++d) + q_rows.push_back(h * hd + rope_source(d, rd)); + LSE_ASSIGN_OR(w_.q_proj, b.require_rows(p + ".q_proj.weight", q_rows, + Shape{qh * hd, hidden})); + + std::vector k_rows; + k_rows.reserve(static_cast(kvh * hd)); + for (std::int64_t h = 0; h < kvh; ++h) + for (std::int64_t d = 0; d < hd; ++d) + k_rows.push_back(h * hd + rope_source(d, rd)); + LSE_ASSIGN_OR(w_.k_proj, b.require_rows(p + ".k_proj.weight", k_rows, + Shape{kvh * hd, hidden})); + + LSE_ASSIGN_OR(w_.v_proj, b.require(p + ".v_proj.weight")); + LSE_RETURN_IF_ERROR(qwen3_5::expect_shape( + w_.v_proj, p + ".v_proj.weight", Shape{kvh * hd, hidden})); + LSE_ASSIGN_OR(w_.o_proj, b.require(p + ".o_proj.weight")); + LSE_RETURN_IF_ERROR(qwen3_5::expect_shape( + w_.o_proj, p + ".o_proj.weight", Shape{hidden, qh * hd})); + + // The HF gate is one scalar per head ([q_heads, hidden]). Expanding each + // head row across its head_dim channels at load turns it into the + // per-channel gate GatedAttention multiplies in — identical math, since + // merged channel h*hd+d belongs to head h for every d. + std::vector g_rows; + g_rows.reserve(static_cast(qh * hd)); + for (std::int64_t h = 0; h < qh; ++h) + for (std::int64_t d = 0; d < hd; ++d) g_rows.push_back(h); + LSE_ASSIGN_OR(w_.g_proj, b.require_rows(p + ".gate_proj.weight", g_rows, + Shape{qh * hd, hidden})); + + // The head norms run before the rotation, so their channel order follows + // the same projection reordering. + std::vector norm_rows; + norm_rows.reserve(static_cast(hd)); + for (std::int64_t d = 0; d < hd; ++d) + norm_rows.push_back(rope_source(d, rd)); + LSE_ASSIGN_OR(w_.q_norm, + b.require_rows(p + ".q_norm.weight", norm_rows, Shape{hd})); + LSE_ASSIGN_OR(w_.k_norm, + b.require_rows(p + ".k_norm.weight", norm_rows, Shape{hd})); + + spec_.q_heads = static_cast(qh); + spec_.kv_heads = static_cast(kvh); + spec_.head_dim = c.attn_head_dim; + spec_.gate = ops::GateSource::kSeparateProj; + spec_.mask = sliding ? graph::MaskKind::kSlidingWindow + : graph::MaskKind::kCausal; + spec_.window = sliding ? static_cast(c.sliding_window) : 0; + spec_.norm_eps = c.rms_eps; + spec_.zero_centered_norm = false; + spec_.kv_length = c.kv_capacity(); + spec_.kv_cache_dtype = c.kv_cache_dtype; + spec_.sparse_attention = c.sparse_attention; + LSE_ASSIGN_OR(rope_, ops::build_rope(static_cast(rd), c.kv_capacity(), theta)); + + auto post = b.optional(std::string(prefix) + + ".post_operator_norm.weight"); + if (post.ok()) post_norm_ = post.release(); + return OkStatus(); + } + + Result forward(const Array& x, MixerState* state, + const LayerContext& ctx) override { + LSE_ASSIGN_OR(Array y, attend(x, state, ctx.attention_phase)); + if (post_norm_.valid()) + y = ops::rms_norm(y, post_norm_, ctx.config->rms_eps); + return y; + } + + private: + Result attend(const Array& x, MixerState* state, + ops::AttentionExecutionPhase phase) { + if (state == nullptr) { + return ops::gated_attention(x, w_, spec_, rope_, 0, nullptr, phase); + } + MixerState& st = *state; + ops::AttentionCache cache; + cache.keys = st.key_cache; + cache.values = st.value_cache; + cache.table = st.paged.table; + cache.meta = st.kv_meta; + cache.paged = &st.paged; + cache.capacity = spec_.kv_length; + cache.used = st.position; + cache.tree_positions = st.tree_positions; + LSE_ASSIGN_OR(Array y, ops::gated_attention(x, w_, spec_, rope_, + st.position, &cache, phase)); + st.key_cache = cache.keys; + st.value_cache = cache.values; + st.tree_keys = cache.tree_keys; + st.tree_values = cache.tree_values; + return y; + } + + ops::GatedAttentionWeights w_; + ops::GatedAttentionSpec spec_; + ops::RopeTables rope_; + Array post_norm_; +}; + +class Lfm3Conv final : public IMixer { + public: + static constexpr std::string_view kName = "lfm3.conv"; + std::string_view name() const noexcept override { return kName; } + + Status load(WeightBinder& b, std::string_view prefix, + const LayerContext& ctx) override { + const Config& c = *ctx.config; + const std::string p = std::string(prefix) + ".conv"; + hidden_ = static_cast(c.hidden_size); + const auto kernel = static_cast(c.gdn_conv_kernel); + + LSE_ASSIGN_OR(in_proj_, b.require(p + ".in_proj.weight")); + LSE_RETURN_IF_ERROR( + qwen3_5::expect_shape(in_proj_, p + ".in_proj.weight", + Shape{3 * hidden_, hidden_})); + // MLX stores the depthwise filter [channels, kernel, 1]; the trailing + // axis is the group-count of 1, so require_as flattens it to [C, K]. + LSE_ASSIGN_OR(conv_w_, + b.require_as(p + ".conv.weight", Shape{hidden_, kernel})); + LSE_ASSIGN_OR(out_proj_, b.require(p + ".out_proj.weight")); + LSE_RETURN_IF_ERROR(qwen3_5::expect_shape( + out_proj_, p + ".out_proj.weight", Shape{hidden_, hidden_})); + + // conv_bias=True checkpoints (LFM2 style) bias all three projections — + // none of the published LFM3 checkpoints use it, so it fails here rather + // than silently dropping a term. + for (const char* name : {".in_proj.bias", ".conv.bias", ".out_proj.bias"}) { + auto bias = b.optional(p + name); + if (bias.ok()) { + return LSE_ERROR(kUnimplemented, "'", std::string(prefix), + "' sets conv_bias; lfm3 convs are read bias-free"); + } + } + // The conv kernel reads weight and bias through one element type — + // the zero pad has to carry the filter's dtype or the bind refuses. + bias_ = Array::zeros(Shape{hidden_}, conv_w_.dtype()); + + auto post = b.optional(std::string(prefix) + + ".post_operator_norm.weight"); + if (post.ok()) post_norm_ = post.release(); + return OkStatus(); + } + + Result forward(const Array& x, MixerState* state, + const LayerContext& ctx) override { + const Array bcx = graph::linear(x, in_proj_); + const Array gate_b = graph::slice(bcx, -1, 0, hidden_); + const Array gate_c = graph::slice(bcx, -1, hidden_, 2 * hidden_); + const Array xv = graph::slice(bcx, -1, 2 * hidden_, 3 * hidden_); + // The convolved stream; state tails are FP32 so the mix happens in F32. + const Array h = graph::cast(graph::mul(gate_b, xv), DType::kF32); + + Array y; + if (state != nullptr && state->gdn_conv_qkv.valid()) { + const Array prev = state->gdn_conv_qkv; + if (state->pad_tail_rows.valid()) { + // A padded pass: the tail is the last kernel-1 real rows, wherever + // the pass's last real token falls — never the pad columns. + const Shape& sx = h.shape(); + const Array flat = graph::reshape(h, Shape{sx.dim(0) * sx.dim(1), sx.dim(2)}); + state->gdn_conv_qkv = graph::reshape( + graph::gather_rows(flat, state->pad_tail_rows), prev.shape()); + } else { + state->gdn_conv_qkv = graph::conv_tail(prev, h); + } + y = graph::causal_conv1d(h, conv_w_, bias_, prev); + } else { + y = graph::causal_conv1d(h, conv_w_, bias_); + } + if (y.dtype() != gate_c.dtype()) y = graph::cast(y, gate_c.dtype()); + y = graph::mul(gate_c, y); + y = graph::linear(y, out_proj_); + if (post_norm_.valid()) + y = ops::rms_norm(y, post_norm_, ctx.config->rms_eps); + return y; + } + + private: + std::int64_t hidden_ = 0; + Array in_proj_, conv_w_, out_proj_, bias_, post_norm_; +}; + +class Lfm3Mlp final : public IFeedForward { + public: + static constexpr std::string_view kName = "lfm3.mlp"; + std::string_view name() const noexcept override { return kName; } + + Status load(WeightBinder& b, std::string_view prefix, + const LayerContext& ctx) override { + const Config& c = *ctx.config; + const std::string p = std::string(prefix) + ".feed_forward"; + const auto hidden = static_cast(c.hidden_size); + const auto inter = static_cast(c.mlp_intermediate); + + LSE_ASSIGN_OR(w1_, b.require(p + ".w1.weight")); + LSE_RETURN_IF_ERROR(qwen3_5::expect_shape(w1_, p + ".w1.weight", + Shape{inter, hidden})); + LSE_ASSIGN_OR(w3_, b.require(p + ".w3.weight")); + LSE_RETURN_IF_ERROR(qwen3_5::expect_shape(w3_, p + ".w3.weight", + Shape{inter, hidden})); + LSE_ASSIGN_OR(w2_, b.require(p + ".w2.weight")); + LSE_RETURN_IF_ERROR(qwen3_5::expect_shape(w2_, p + ".w2.weight", + Shape{hidden, inter})); + + auto post = b.optional(std::string(prefix) + ".post_ffn_norm.weight"); + if (post.ok()) post_norm_ = post.release(); + return OkStatus(); + } + + Result forward(const Array& x, Array* aux_loss, + const LayerContext& ctx) override { + (void)aux_loss; + Array y = ops::swiglu(x, w1_, w3_, w2_); + if (post_norm_.valid()) + y = ops::rms_norm(y, post_norm_, ctx.config->rms_eps); + return y; + } + + private: + Array w1_, w3_, w2_, post_norm_; +}; + +HybridBlockSpec block_spec() { + return HybridBlockSpec{".operator_norm.weight", ".ffn_norm.weight"}; +} + +// The conv tail rides the GDN channel-width slot: every conv layer gets a +// [B, kernel-1, hidden] window of its own conv input. gdn_state stays zero — +// there is no recurrent state anywhere in the stack. +HybridLMSpec lm_spec(const Config& config) { + HybridLMSpec spec; + spec.embed_name = "language_model.model.embed_tokens.weight"; + // Despite the name this is the model's final norm — modeling_lfm3 applies + // it to the last hidden state, not to the embedding output. + spec.final_norm_name = "language_model.model.embedding_norm.weight"; + spec.block_prefix = std::string(kBlockPrefix); + spec.lm_head_name = + config.tie_word_embeddings ? "" : "language_model.lm_head.weight"; + spec.zero_centered_norm = false; + spec.gdn_conv_width = config.hidden_size; + spec.gdn_state_heads = 0; + spec.gdn_state_dim = 0; + // Omni checkpoints carry non-text towers beside this stack; refusing them + // is what keeps text decode honest about what it cannot encode. + spec.refused.push_back( + {"vision_tower.", "this build decodes text only; the Lfm3Omni vision " + "tower is not implemented, so image tokens cannot be " + "encoded"}); + spec.refused.push_back( + {"audio_tower.", "this build decodes text only; the Lfm3Omni audio " + "tower is not implemented, so audio tokens cannot be " + "encoded"}); + spec.refused.push_back( + {"audio_projector.", "this build decodes text only; the audio " + "projector feeds the refused audio tower"}); + spec.refused.push_back( + {"depth_decoder.", "this build decodes text only; the depth decoder " + "head is not implemented"}); + spec.refused.push_back( + {"multi_modal_projector.", "this build decodes text only; the " + "multimodal projector feeds refused towers"}); + spec.refused.push_back( + {"codebook_embedding.", "this build decodes text only; the audio " + "codebook table is not implemented"}); + return spec; +} + +// Every LFM3 layer 0 is a conv layer carrying conv.conv.weight, and the +// model-level norm is embedding_norm — together they separate this stack +// from qwen3_5 (linear_attn.*, norm.weight) and lemonseed (blocks.N.mixer.*) +// regardless of how similar the layer schedules look. +bool looks_like_lfm3(const Config& config, const SafeTensors& weights) { + (void)config; + const std::string block = std::string(kBlockPrefix) + ".0.conv"; + return weights.find(block + ".conv.weight") != nullptr && + weights.find(block + ".in_proj.weight") != nullptr && + weights.find(block + ".out_proj.weight") != nullptr && + weights.find("language_model.model.embedding_norm.weight") != + nullptr; +} + +std::unique_ptr make_lfm3(const Config& config) { + return std::make_unique( + config, lm_spec(config), + [config](std::int32_t layer) -> Result> { + std::unique_ptr mixer = + config.is_attention_layer(layer) + ? std::unique_ptr(std::make_unique()) + : std::unique_ptr(std::make_unique()); + return std::make_unique( + std::move(mixer), std::make_unique(), + /*zero_centered_norm=*/false, /*mod=*/nullptr, block_spec()); + }); +} + +} // namespace + +} // namespace lse::model + +LSE_REGISTER_MODEL(::lse::model::ModelArch{ + "lfm3", 0, &::lse::model::looks_like_lfm3, + [](const ::lse::model::Config& c) { return ::lse::model::make_lfm3(c); }}) diff --git a/src/server/chat_protocol.cpp b/src/server/chat_protocol.cpp index a79ae642..39ee183b 100644 --- a/src/server/chat_protocol.cpp +++ b/src/server/chat_protocol.cpp @@ -183,6 +183,8 @@ ChatRequest prepare_chat(const ChatJson& body, const ChatFraming& framing) { if (!system.empty()) { if (!prefix.empty()) prefix += "\n\n"; prefix += system; } if (!prefix.empty()) messages.insert(messages.begin(), {"system", prefix}); result.prompt = render_chatml(messages, false) + framing.generation_prompt; + if (std::getenv("LSE_DUMP_PROMPT") != nullptr) + std::fprintf(stderr, "LSE-CHAT-PROMPT>>>%s<<<\n", result.prompt.c_str()); return result; } diff --git a/src/server/router.cpp b/src/server/router.cpp index 1c3b23e7..22df6bfe 100644 --- a/src/server/router.cpp +++ b/src/server/router.cpp @@ -1010,6 +1010,14 @@ void Router::Impl::completion(bool chat, std::string_view req_body, return; } r.prompt = *encoded; + // Chat-formatted prompts carry the tokenizer's declared BOS, matching + // HF/vLLM's add_special_tokens behavior on the same path. The completions + // passthrough stays verbatim — no BOS — as llama.cpp/MLX/vLLM treat it. + if (chat) { + const std::uint32_t bos = impl.tok.bos_token_id(); + if (bos != 0 && (r.prompt.empty() || r.prompt.front() != bos)) + r.prompt.insert(r.prompt.begin(), bos); + } // The prompt must leave room for at least one generated token. const int context_length = impl.model.config().kv_capacity(); diff --git a/src/tokenizer/tokenizer.cpp b/src/tokenizer/tokenizer.cpp index 1c3e1b53..89aed5bf 100644 --- a/src/tokenizer/tokenizer.cpp +++ b/src/tokenizer/tokenizer.cpp @@ -1,8 +1,11 @@ #include "lse/tokenizer/tokenizer.hpp" #include +#include #include +#include + #include "fastokens_ffi.h" namespace lse::tokenizer { @@ -25,12 +28,14 @@ Tokenizer::~Tokenizer() { } Tokenizer::Tokenizer(Tokenizer&& other) noexcept - : handle_(std::exchange(other.handle_, nullptr)) {} + : handle_(std::exchange(other.handle_, nullptr)), + bos_id_(std::exchange(other.bos_id_, 0)) {} Tokenizer& Tokenizer::operator=(Tokenizer&& other) noexcept { if (this != &other) { if (handle_) fk_free(as_tok(handle_)); handle_ = std::exchange(other.handle_, nullptr); + bos_id_ = std::exchange(other.bos_id_, 0); } return *this; } @@ -43,6 +48,27 @@ Result Tokenizer::from_file(const std::string& tokenizer_json) { } Tokenizer out; out.handle_ = t; + // tokenizer_config.json sits beside tokenizer.json; a declared bos_token + // marks checkpoints that expect a leading BOS the post-processor never adds. + // A missing or malformed config costs the marker, not the load. + std::ifstream in(fs::path(tokenizer_json).parent_path() / + "tokenizer_config.json"); + if (in) { + try { + const nlohmann::json cfg = nlohmann::json::parse(in); + std::string name; + if (const auto it = cfg.find("bos_token"); it != cfg.end()) { + if (it->is_string()) name = it->get(); + else if (it->is_object()) + if (const auto c = it->find("content"); + c != it->end() && c->is_string()) + name = c->get(); + } + if (!name.empty()) + if (auto id = out.token_to_id(name); id.ok()) out.bos_id_ = *id; + } catch (...) { + } + } return out; } diff --git a/tests/test_model.cpp b/tests/test_model.cpp index bcf139b9..560a1c73 100644 --- a/tests/test_model.cpp +++ b/tests/test_model.cpp @@ -681,6 +681,15 @@ const std::vector& qwen_moe_names() { return v; } +const std::vector& lfm3_names() { + static const std::vector v{ + "language_model.model.layers.0.conv.conv.weight", + "language_model.model.layers.0.conv.in_proj.weight", + "language_model.model.layers.0.conv.out_proj.weight", + "language_model.model.embedding_norm.weight"}; + return v; +} + // Detected name, or "" when nothing claims the fixture. std::string detect(const std::string& stem, const std::vector& names) { auto st = SafeTensors::open(write_fixture(stem, names)); @@ -697,6 +706,7 @@ LSE_TEST(every_architecture_claims_only_its_own_checkpoint) { LSE_EXPECT(detect("lemonseed", lemonseed_names()) == "lemonseed"); LSE_EXPECT(detect("q35", qwen_dense_names()) == "qwen3.5"); LSE_EXPECT(detect("q35moe", qwen_moe_names()) == "qwen3.5-moe"); + LSE_EXPECT(detect("lfm3", lfm3_names()) == "lfm3"); } LSE_TEST(qwen_predicates_cannot_both_claim_a_checkpoint) { @@ -734,6 +744,7 @@ LSE_TEST(an_unrecognized_checkpoint_names_what_was_tried) { LSE_EXPECT(msg.find("lemonseed") != std::string::npos); LSE_EXPECT(msg.find("qwen3.5") != std::string::npos); LSE_EXPECT(msg.find("qwen3.5-moe") != std::string::npos); + LSE_EXPECT(msg.find("lfm3") != std::string::npos); } // --- the two weight reorderings the Qwen kernels rely on --------------------- @@ -1069,6 +1080,59 @@ LSE_TEST(layer_types_must_agree_with_the_interval_rule) { LSE_EXPECT(Config::from_json_string(hf_config(good)).ok()); } +LSE_TEST(lfm3_config_is_read_with_per_type_rope) { + std::string lfm_cfg = R"({ + "model_type": "lfm3", + "dtype": "bfloat16", + "vocab_size": 32000, + "hidden_size": 2048, + "num_hidden_layers": 4, + "norm_eps": 1e-5, + "num_attention_heads": 16, + "num_key_value_heads": 4, + "head_dim": 128, + "intermediate_size": 5632, + "conv_L_cache": 7, + "layer_types": ["conv", "full_attention", "conv", "sliding_attention"], + "rope_parameters": { + "full_attention": {"rope_theta": 10000.0, "partial_rotary_factor": 1.0}, + "sliding_attention": {"rope_theta": 1000000.0, "partial_rotary_factor": 0.5} + }, + "sliding_window": 512, + "max_position_embeddings": 32768 + })"; + auto c = Config::from_json_string(lfm_cfg); + LSE_EXPECT(c.ok()); + if (!c.ok()) return; + LSE_EXPECT_EQ(c->vocab_size, 32000); + LSE_EXPECT_EQ(c->hidden_size, 2048); + LSE_EXPECT_EQ(c->num_layers, 4); + LSE_EXPECT_EQ(c->attn_q_heads, 16); + LSE_EXPECT_EQ(c->attn_kv_heads, 4); + LSE_EXPECT_EQ(c->attn_head_dim, 128); + LSE_EXPECT_EQ(c->rope_dim, 128); + LSE_EXPECT_EQ(c->sliding_rope_dim, 64); + LSE_EXPECT_EQ(c->rope_theta, 10000.0f); + LSE_EXPECT_EQ(c->sliding_rope_theta, 1000000.0f); + LSE_EXPECT_EQ(c->gdn_conv_kernel, 7); + LSE_EXPECT_EQ(c->sliding_window, 512); + LSE_EXPECT(!c->is_attention_layer(0)); + LSE_EXPECT(c->is_attention_layer(1)); + LSE_EXPECT(!c->is_sliding_attention(1)); + LSE_EXPECT(!c->is_attention_layer(2)); + LSE_EXPECT(c->is_attention_layer(3)); + LSE_EXPECT(c->is_sliding_attention(3)); + LSE_EXPECT(c->global_attention_layers.empty()); + LSE_EXPECT(!c->is_global_attention(1)); + + auto rt = Config::from_json_string(c->to_json()); + LSE_EXPECT(rt.ok()); + if (!rt.ok()) return; + LSE_EXPECT_EQ(rt->sliding_rope_dim, 64); + LSE_EXPECT_EQ(rt->sliding_rope_theta, 1000000.0f); + LSE_EXPECT_EQ(static_cast(rt->layer_types.size()), 4); +} + LSE_TEST(a_lemonseed_config_still_takes_the_native_path) { // No text_config, so none of the HF requirements apply. const Config a = preset_b1_5();