Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,7 @@ add_library(lse_model
src/models/qwen3_5.cpp
src/models/qwen3_5_common.cpp
src/models/qwen3_5_moe.cpp
src/models/lfm3.cpp
)
add_library(lse::model ALIAS lse_model)
# Model kernels self-register from a TU nothing references.
Expand Down
20 changes: 20 additions & 0 deletions include/lse/model/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,16 @@ struct Config {
std::int32_t attn_head_dim = 64;
std::int32_t rope_dim = 64;
float rope_theta = 10000.0f;
// RoPE profile for `sliding_attention` layers when it differs from full
// attention's (LFM3: theta 1e4 full-width vs 1e6 half-width). Zero/absent
// means sliding layers share the primary profile.
std::int32_t sliding_rope_dim = 0;
float sliding_rope_theta = 0.0f;
// The checkpoint's own layer schedule ("conv", "linear_attention",
// "full_attention", "sliding_attention"). Authoritative for mixer choice
// when populated; full_attention_interval is the fallback for configs that
// never declare it.
std::vector<std::string> layer_types;

// Gated DeltaNet
std::int32_t gdn_qk_heads = 4;
Expand Down Expand Up @@ -120,9 +130,19 @@ struct Config {
}

[[nodiscard]] bool is_attention_layer(std::int32_t layer_idx) const noexcept {
if (!layer_types.empty()) {
const std::string& t = layer_types[static_cast<std::size_t>(layer_idx)];
return t == "full_attention" || t == "sliding_attention";
}
return (layer_idx + 1) % full_attention_interval == 0;
}

[[nodiscard]] bool is_sliding_attention(std::int32_t layer_idx) const noexcept {
return !layer_types.empty() &&
layer_types[static_cast<std::size_t>(layer_idx)] ==
"sliding_attention";
}

[[nodiscard]] bool is_global_attention(std::int32_t layer_idx) const noexcept {
for (std::int32_t l : global_attention_layers) {
if (l == layer_idx) return true;
Expand Down
6 changes: 6 additions & 0 deletions include/lse/tokenizer/tokenizer.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,17 @@ class Tokenizer {

[[nodiscard]] bool valid() const noexcept { return handle_ != nullptr; }
[[nodiscard]] void* raw() const noexcept { return handle_; }
// The id tokenizer_config.json declares as bos_token, 0 when none is
// declared. GPT-2-family checkpoints (LFM3) expect a leading BOS that the
// ByteLevel post-processor never injects — chat prompts prepend it to match
// HF/vLLM's add_special_tokens behavior; raw completions stay verbatim.
[[nodiscard]] std::uint32_t bos_token_id() const noexcept { return bos_id_; }

DecodeStream stream(bool skip_special_tokens = true) const;

private:
void* handle_ = nullptr;
std::uint32_t bos_id_ = 0;
};

// Incremental detokenization for streaming. Returns text only once a token
Expand Down
8 changes: 4 additions & 4 deletions src/kernels/sdpa.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1212,7 +1212,7 @@ struct SdpaKernel final : KernelPrimitive<SdpaKernel> {
} else if (mask == 1) {
if (auto g = e.when(j <= abs_i)) take();
} else {
if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) take();
if (auto g = e.when(j <= abs_i && j + window > abs_i)) take();
}
}

Expand All @@ -1232,7 +1232,7 @@ struct SdpaKernel final : KernelPrimitive<SdpaKernel> {
} else if (mask == 1) {
if (auto g = e.when(j <= abs_i)) apply();
} else {
if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) apply();
if (auto g = e.when(j <= abs_i && j + window > abs_i)) apply();
}
denom = denom.read() + w.read();
const auto vb = ((b * kvh + kh) * ts + j) * dv + d;
Expand Down Expand Up @@ -1290,7 +1290,7 @@ struct SdpaKernel final : KernelPrimitive<SdpaKernel> {
} else if (mask == 3) {
if (auto g = e.when(tree_visible(j))) take();
} else {
if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) take();
if (auto g = e.when(j <= abs_i && j + window > abs_i)) take();
}
}
}
Expand Down Expand Up @@ -1324,7 +1324,7 @@ struct SdpaKernel final : KernelPrimitive<SdpaKernel> {
} else if (mask == 3) {
if (auto g = e.when(tree_visible(j))) apply();
} else {
if (auto g = e.when(j <= abs_i && (abs_i - j) < window)) apply();
if (auto g = e.when(j <= abs_i && j + window > abs_i)) apply();
}
denom = denom.read() + w.read();
acc = math::fma(w.read(), kv_load<Storage>(e, a.v, dim_major ? e.let(vb0 + jj)
Expand Down
152 changes: 148 additions & 4 deletions src/model/config.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "lse/model/config.hpp"

#include <algorithm>
#include <fstream>
#include <sstream>
#include <utility>
Expand Down Expand Up @@ -71,6 +72,111 @@ Status read_storage_config(const nlohmann::json& root, Config& c) {
return OkStatus();
}

// LFM3's text stack (Lfm3ForCausalLM, and the language model inside the
// Lfm3Omni wrapper), read off LiquidAI/LFM3-1.7B-MLX-4bit and
// LiquidAI/LFM3-Omni-2B-MLX-4bit and cross-checked against
// transformers/models/lfm3/modeling_lfm3.py. A conv/attention hybrid:
// `conv` layers are a gated depthwise width-7 convolution, attention layers
// carry per-head q/k norms, a per-head sigmoid output gate and a
// layer-type rotary profile.
Result<Config> from_lfm3_json(const nlohmann::json& root) {
const auto nested = root.find("text_config");
const nlohmann::json& t =
nested != root.end() && nested->is_object() ? *nested : root;
Config c;

LSE_RETURN_IF_ERROR(need(t, "vocab_size", c.vocab_size));
LSE_RETURN_IF_ERROR(need(t, "hidden_size", c.hidden_size));
LSE_RETURN_IF_ERROR(need(t, "num_hidden_layers", c.num_layers));
LSE_RETURN_IF_ERROR(need(t, "norm_eps", c.rms_eps));
LSE_RETURN_IF_ERROR(need(t, "num_attention_heads", c.attn_q_heads));
LSE_RETURN_IF_ERROR(need(t, "num_key_value_heads", c.attn_kv_heads));
LSE_RETURN_IF_ERROR(need(t, "head_dim", c.attn_head_dim));
LSE_RETURN_IF_ERROR(need(t, "intermediate_size", c.mlp_intermediate));
// The conv mixer's kernel width; the stack carries the kernel-1 tail.
LSE_RETURN_IF_ERROR(need(t, "conv_L_cache", c.gdn_conv_kernel));

// layer_types is the whole schedule: `conv` gets the short conv,
// `full_attention`/`sliding_attention` get attention.
if (!t.contains("layer_types") || !t["layer_types"].is_array()) {
return LSE_ERROR(kNotFound, "model config is missing 'layer_types'");
}
c.layer_types = t["layer_types"].get<std::vector<std::string>>();
if (static_cast<std::int32_t>(c.layer_types.size()) != c.num_layers) {
return LSE_ERROR(kInvalidArgument, "layer_types has ",
std::to_string(c.layer_types.size()), " entries but "
"num_hidden_layers is ", std::to_string(c.num_layers));
}
for (const std::string& type : c.layer_types) {
if (type != "conv" && type != "full_attention" &&
type != "sliding_attention") {
return LSE_ERROR(kUnimplemented, "layer_types names '", type,
"'; lfm3 mixers are conv, full_attention or "
"sliding_attention");
}
}

// Each attention kind owns a rotary profile: theta plus how much of the
// head rotates. Sliding defaults to full's when unlisted.
const auto ropes = t.find("rope_parameters");
if (ropes == t.end() || !ropes->is_object()) {
return LSE_ERROR(kNotFound, "model config is missing 'rope_parameters'");
}
auto read_rope = [&c](const nlohmann::json& r, float& theta,
std::int32_t& dim) -> Status {
LSE_RETURN_IF_ERROR(need(r, "rope_theta", theta));
float partial = 1.0f;
read(r, "partial_rotary_factor", partial);
const auto rotary = static_cast<std::int32_t>(
static_cast<float>(c.attn_head_dim) * partial + 0.5f);
if (rotary <= 0 || rotary % 2 != 0 || rotary > c.attn_head_dim) {
return LSE_ERROR(kInvalidArgument, "partial_rotary_factor ",
std::to_string(partial), " of head_dim ",
std::to_string(c.attn_head_dim), " gives a rope width "
"of ", std::to_string(rotary),
", which must be even and positive");
}
dim = rotary;
return OkStatus();
};
if (!ropes->contains("full_attention")) {
return LSE_ERROR(kNotFound,
"rope_parameters is missing 'full_attention'");
}
LSE_RETURN_IF_ERROR(
read_rope((*ropes)["full_attention"], c.rope_theta, c.rope_dim));
c.sliding_rope_theta = c.rope_theta;
c.sliding_rope_dim = c.rope_dim;
if (ropes->contains("sliding_attention") &&
(*ropes)["sliding_attention"].is_object()) {
LSE_RETURN_IF_ERROR(read_rope((*ropes)["sliding_attention"],
c.sliding_rope_theta, c.sliding_rope_dim));
}

read(t, "sliding_window", c.sliding_window);

// No delta-net layers, no experts: the non-attention mixer is the conv.
c.gdn_qk_heads = 0;
c.gdn_v_heads = 0;
c.gdn_head_dim = 0;
c.num_experts = 0;
c.num_active_experts = 0;
c.num_shared_experts = 0;
c.expert_intermediate = 0;
c.mtp_layers = 0;
// layer_types is authoritative; the interval rule is never consulted.
c.full_attention_interval = 1;
c.global_attention_layers.clear();

read(root, "tie_word_embeddings", c.tie_word_embeddings);
read(t, "tie_word_embeddings", c.tie_word_embeddings);
LSE_RETURN_IF_ERROR(read_storage_config(root, c));
read(t, "max_position_embeddings", c.train_seq_len);

LSE_RETURN_IF_ERROR(c.validate());
return c;
}

// Qwen3.5 (dense and MoE), read off Qwen/Qwen3.5-0.8B, Qwen/Qwen3.5-4B and
// Qwen/Qwen3.5-35B-A3B and cross-checked against
// transformers/models/qwen3_5/configuration_qwen3_5.py. Both are gated
Expand Down Expand Up @@ -195,7 +301,8 @@ Result<Config> from_hf_json(const nlohmann::json& root) {
"num_hidden_layers is ", std::to_string(c.num_layers));
}
for (std::size_t i = 0; i < types.size(); ++i) {
const bool attends = types[i] == "full_attention";
const bool attends = types[i] == "full_attention" ||
types[i] == "sliding_attention";
if (attends != c.is_attention_layer(static_cast<std::int32_t>(i))) {
return LSE_ERROR(kUnimplemented, "layer_types says layer ",
std::to_string(i), " is '", types[i],
Expand All @@ -204,6 +311,7 @@ Result<Config> from_hf_json(const nlohmann::json& root) {
" says otherwise");
}
}
c.layer_types = types;
}

LSE_RETURN_IF_ERROR(c.validate());
Expand All @@ -229,6 +337,29 @@ Result<Config> Config::from_json_string(const std::string& text) {
LSE_ASSIGN_OR(quant::GroupAffineMap quantization,
quant::GroupAffineMap::from_config_json(text));

// LFM3 (and Lfm3Omni's text stack) parses differently enough to own a
// branch: its config is HF-shaped but flat-or-nested depending on whether
// the checkpoint is the wrapper, and its rope_parameters are keyed per
// layer type rather than global.
std::string model_type;
if (j.contains("model_type") && j["model_type"].is_string())
model_type = j["model_type"].get<std::string>();
if (model_type.empty() && j.contains("text_config") &&
j["text_config"].is_object() &&
j["text_config"].contains("model_type") &&
j["text_config"]["model_type"].is_string())
model_type = j["text_config"]["model_type"].get<std::string>();
std::string arch0;
if (j.contains("architectures") && j["architectures"].is_array() &&
!j["architectures"].empty() && j["architectures"][0].is_string())
arch0 = j["architectures"][0].get<std::string>();
if (model_type.rfind("lfm", 0) == 0 || arch0.rfind("Lfm", 0) == 0) {
LSE_ASSIGN_OR(Config lfm, from_lfm3_json(j));
lfm.quantization = std::move(quantization);
lfm.sampling_defaults = sampling;
return lfm;
}

if (j.contains("text_config") && j["text_config"].is_object()) {
LSE_ASSIGN_OR(Config hf, from_hf_json(j));
hf.quantization = std::move(quantization);
Expand All @@ -251,6 +382,9 @@ Result<Config> Config::from_json_string(const std::string& text) {
read(j, "attn_head_dim", c.attn_head_dim);
read(j, "rope_dim", c.rope_dim);
read(j, "rope_theta", c.rope_theta);
read(j, "sliding_rope_dim", c.sliding_rope_dim);
read(j, "sliding_rope_theta", c.sliding_rope_theta);
read(j, "layer_types", c.layer_types);

read(j, "gdn_qk_heads", c.gdn_qk_heads);
read(j, "gdn_v_heads", c.gdn_v_heads);
Expand Down Expand Up @@ -321,6 +455,9 @@ std::string Config::to_json() const {
j["attn_head_dim"] = attn_head_dim;
j["rope_dim"] = rope_dim;
j["rope_theta"] = rope_theta;
j["sliding_rope_dim"] = sliding_rope_dim;
j["sliding_rope_theta"] = sliding_rope_theta;
j["layer_types"] = layer_types;
j["gdn_qk_heads"] = gdn_qk_heads;
j["gdn_v_heads"] = gdn_v_heads;
j["gdn_head_dim"] = gdn_head_dim;
Expand Down Expand Up @@ -356,16 +493,23 @@ Status Config::validate() const {
return LSE_ERROR(kInvalidArgument, "vocab_size, hidden_size and num_layers "
"must all be positive");
}
if (full_attention_interval <= 0) {
if (full_attention_interval <= 0 && layer_types.empty()) {
return LSE_ERROR(kInvalidArgument, "full_attention_interval must be positive");
}
if (attn_q_heads % attn_kv_heads != 0) {
return LSE_ERROR(kInvalidArgument, "GQA requires attn_q_heads (",
std::to_string(attn_q_heads), ") to be a multiple of "
"attn_kv_heads (", std::to_string(attn_kv_heads), ")");
}
if (gdn_qk_heads <= 0 || gdn_v_heads <= 0 ||
(gdn_qk_heads % gdn_v_heads != 0 && gdn_v_heads % gdn_qk_heads != 0)) {
// Only checked when the stack actually runs delta-net layers — either
// every non-attention layer (layer_types unset, lemonseed's layout) or the
// ones the schedule names. LFM3's conv layers carry no recurrent state.
const bool has_gdn = layer_types.empty() ||
std::find(layer_types.begin(), layer_types.end(),
"linear_attention") != layer_types.end();
if (has_gdn &&
(gdn_qk_heads <= 0 || gdn_v_heads <= 0 ||
(gdn_qk_heads % gdn_v_heads != 0 && gdn_v_heads % gdn_qk_heads != 0))) {
// Either direction: lemonseed's layer runs v at the key width and reads
// gdn_v_heads as a divisor of it, Qwen shares 16 key heads across 32 value
// heads. Both are GQA over the same state, counted from opposite ends.
Expand Down
39 changes: 24 additions & 15 deletions src/model/hybrid_lm.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1085,7 +1085,7 @@ Result<Array> HybridLM::hidden(const Array& tokens,
const graph::ScopedMember on(shards > 1
? i % shards
: member_for_layer(layer));
if (!st.gdn_state.valid()) {
if (vh_m > 0 && vd > 0 && !st.gdn_state.valid()) {
LSE_ASSIGN_OR(st.gdn_state, mutable_zeros(Shape{batch, vh_m, vd, vd}));
}
if (tail <= 0) continue;
Expand Down Expand Up @@ -1525,22 +1525,31 @@ Result<Array> HybridLM::hidden(const Array& tokens,
if (t_live < t_now) {
const auto heads = static_cast<std::int64_t>(config_.gdn_v_heads);
const auto tail = static_cast<std::int64_t>(config_.gdn_conv_kernel) - 1;
if (heads <= 0 || tail <= 0 || t_live < tail)
if (heads <= 0 && tail <= 0)

Copy link
Copy Markdown
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need you to verify the existing Qwen3.8-27B still works with this rule here

return LSE_ERROR(kInvalidArgument, "a padded pass needs at least ",
std::to_string(tail), " real tokens and GDN value heads");
std::vector<float> m(static_cast<std::size_t>(t_now * heads), 0.0f);
std::vector<float> u(m.size(), 1.0f);
for (std::int64_t t = 0; t < t_live; ++t)
for (std::int64_t h = 0; h < heads; ++h) {
m[static_cast<std::size_t>(t * heads + h)] = 1.0f;
u[static_cast<std::size_t>(t * heads + h)] = 0.0f;
}
std::vector<float> rows_of_tail(static_cast<std::size_t>(tail));
for (std::int64_t i = 0; i < tail; ++i)
rows_of_tail[static_cast<std::size_t>(i)] = static_cast<float>(t_live - tail + i);
LSE_ASSIGN_OR(mask, device_values(Shape{1, t_now, heads}, m));
LSE_ASSIGN_OR(unmask, device_values(Shape{1, t_now, heads}, u));
LSE_ASSIGN_OR(tail_rows, device_values(Shape{tail}, rows_of_tail));
if (t_live < tail)
return LSE_ERROR(kInvalidArgument, "a padded pass needs at least ",
std::to_string(tail), " real tokens");
// Per-head write suppression is a GDN-only need; the tail rows are what
// every conv-tail state (GDN or LFM-style) gathers on a padded pass.
if (heads > 0) {
std::vector<float> m(static_cast<std::size_t>(t_now * heads), 0.0f);
std::vector<float> u(m.size(), 1.0f);
for (std::int64_t t = 0; t < t_live; ++t)
for (std::int64_t h = 0; h < heads; ++h) {
m[static_cast<std::size_t>(t * heads + h)] = 1.0f;
u[static_cast<std::size_t>(t * heads + h)] = 0.0f;
}
LSE_ASSIGN_OR(mask, device_values(Shape{1, t_now, heads}, m));
LSE_ASSIGN_OR(unmask, device_values(Shape{1, t_now, heads}, u));
}
if (tail > 0) {
std::vector<float> rows_of_tail(static_cast<std::size_t>(tail));
for (std::int64_t i = 0; i < tail; ++i)
rows_of_tail[static_cast<std::size_t>(i)] = static_cast<float>(t_live - tail + i);
LSE_ASSIGN_OR(tail_rows, device_values(Shape{tail}, rows_of_tail));
}
}
for (MixerState& st : *states) {
st.pad_mask = mask;
Expand Down
Loading
Loading