From e0703eebee02e5d73dd15c79b88028a003238837 Mon Sep 17 00:00:00 2001 From: Daniel Moreno Date: Sun, 27 Sep 2026 17:50:39 -0400 Subject: [PATCH] Fix silent wrong-result lowering; add MobileCLIP2-S2 end-to-end coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Driven by a MobileCLIP2-S2 parity hunt (attention outputs diverged while MLP/norm/downsample matched). Several independent silent-correctness bugs fell out; all are fixed here with coverage in both runners. Correctness fixes (numerics change vs main — all toward the framework reference, so existing exact-GELU / reduction / clamp / SDPA results shift): - GELU: torch exact (approximate='none', the default) lowers to GGML_UNARY_OP_GELU_ERF (16) end to end (importer attribute, lowering, codegen, interpreter); tanh keeps GELU (8). - Single-axis MEAN/SUM over ggml dims 2/3 now swap that axis to position 0 and reduce it in both runners; the old flatten-HxW path reduced the wrong elements. - Multi-axis MEAN/SUM/AMAX/AMIN chain into single-axis ops through the shared ir.graph.chain_single_axis_reduction helper (torch + JAX); lowering rejects unchained multi-axis specs instead of silently using axes[0]. - SDPA without an explicit scale uses 1/sqrt(head_dim) (was 1.0). - CLAMP with an open min/max defaults to -/+FLT_MAX in both runners (was 0/6, silently bounding clamp_min/clamp_max). - Grouped convs with a channel multiplier (groups == in != out) decompose into per-group convs; lowering rejects non-depthwise grouped convs instead of mis-emitting CONV_2D_DW. True depthwise still lowers to one DW op. - JAX TF-SAME convs with asymmetric padding materialize the excess sides as zero concats and keep the symmetric minimum (was: before-side only). - Depthwise convs always use ggml_conv_2d_dw_direct with the mirrored match_dw_layout helper (1D folding, F32 casts) in both runners. Loud failures instead of silent wrong graphs: - Lowering raises NotImplementedError for inexpressible 5D permutes (was: silently a RESHAPE, scrambling QKV head/slot order); the torch importer squeezes static unit dims so QKV-style permutes become plain 4D permutes. - Slices of folded (>4D) dims land on ggml dim 3 with an offset multiplier (was: out-of-range nb[4] in generated code). - Unhandled opcodes in the C++ emitter raise (was: identity passthrough). - Graph arena sized from node count via ggml_new_graph_custom. Compat / plumbing: - GGUF writer shortens tensor names >= GGML_MAX_NAME (64) to name[:54]_sha1[:8], applied consistently to the tensor-info table and the graph-spec JSON the native loader binds weights by. - JAX explicit-pad tensors use padconv_N names (jaxpr brackets/colons broke the GGUF loader); conv bias adds go through match_broadcast (incl. I32->F32 casts); CONCAT supports N inputs and skips empties; flash-attn output is permuted to graph layout unless fused upstream; fused conv+relu; aten.expand_as mapping; unbind/split-container getitem support. - New examples/models/openclip_model.py MobileCLIP2-S2 loader (hf-hub timm checkpoint, reparameterize_model, normalizing wrapper, preprocessor) plus open_clip_torch dev dependency and a CPU e2e parity test. Tests: per-dim MEAN/SUM, chained multi-dim reductions, exact/tanh GELU, QKV 5D permute, SDPA default scale, conv bias/depthwise/grouped-multiplier, clamp_min, expand_as, folded-slice views, GGUF long names — in both the standalone (codegen) and interpreter runners — plus importer/lowering structure tests and JAX reduction/padding tests. --- examples/models/openclip_model.py | 84 +++++++ pyproject.toml | 1 + python/ggmlc/codegen/cpp.py | 200 +++++++++++++-- python/ggmlc/dialect/ggml/lowering.py | 52 +++- python/ggmlc/dialect/ggml/ops.py | 1 + python/ggmlc/frontend/jax/importer.py | 97 +++++++- python/ggmlc/frontend/pytorch/importer.py | 218 ++++++++++++---- python/ggmlc/frontend/pytorch/operators.py | 2 +- python/ggmlc/ir/graph.py | 50 +++- python/ggmlc/ir/op.py | 1 - python/ggmlc/serialization/gguf.py | 41 ++- runtime/src/executor.cpp | 51 ++-- tests/codegen/test_standalone_execution.py | 233 ++++++++++++++++++ tests/e2e/test_openclip_models.py | 45 ++++ .../frontend/test_importer_decompositions.py | 167 +++++++++++++ tests/ops/test_folded_slice.py | 70 ++++++ tests/ops/test_grouped_conv.py | 51 ++++ tests/runtime/test_interpreter_parity.py | 196 +++++++++++++++ .../serialization/test_gguf_serialization.py | 29 +++ tests/unit/test_frontend_jax.py | 49 ++++ 20 files changed, 1536 insertions(+), 102 deletions(-) create mode 100644 examples/models/openclip_model.py create mode 100644 tests/e2e/test_openclip_models.py create mode 100644 tests/frontend/test_importer_decompositions.py create mode 100644 tests/ops/test_folded_slice.py create mode 100644 tests/ops/test_grouped_conv.py create mode 100644 tests/runtime/test_interpreter_parity.py diff --git a/examples/models/openclip_model.py b/examples/models/openclip_model.py new file mode 100644 index 0000000..26d6da1 --- /dev/null +++ b/examples/models/openclip_model.py @@ -0,0 +1,84 @@ +"""Real checkpoint loaders for OpenCLIP models""" + +from __future__ import annotations + +import torch +from ggmlc.pipeline import VisionPreprocessor +from open_clip import create_model_from_pretrained +from open_clip.transform import MaybeConvertMode, MaybeToTensor +from timm.utils import reparameterize_model +from torchvision.transforms.transforms import CenterCrop, Normalize, Resize + +__all__ = [ + "load_openclip_image_encoder_model", +] + +# preprocess +# Compose( +# Resize(size=256, interpolation=bicubic, max_size=None, antialias=True) +# CenterCrop(size=(256, 256)) +# MaybeConvertMode() +# MaybeToTensor() +# Normalize(mean=(0.48145466, 0.4578275, 0.40821073), +# std=(0.26862954, 0.26130258, 0.27577711)) +# ) + +def make_vision_preprocessor(preprocess) -> VisionPreprocessor: + """Creates a GGMLC VisionPreprocessor.""" + kwargs = {} + for transform in preprocess.transforms: + if isinstance(transform, Resize): + target_size = (transform.size, transform.size) \ + if isinstance(transform.size, int) else transform.size + kwargs["target_size"] = target_size + elif isinstance(transform, Normalize): + kwargs["mean"] = transform.mean + kwargs["std"] = transform.std + elif isinstance(transform, (CenterCrop, MaybeConvertMode, MaybeToTensor)): + continue # Ignore + else: + raise TypeError(f"Unsupported transform: {transform}") + return VisionPreprocessor(**kwargs) + + +def load_openclip_image_encoder_model( + variant: str = "hf-hub:timm/MobileCLIP2-S2-OpenCLIP", + with_preprocessor: bool = True, + **kwargs +) -> tuple[torch.nn.Module, tuple[torch.Tensor, ...], list[str], VisionPreprocessor]: + """Loads OpenCLIP pretrained image encoder model. + """ + + if "return_transform" in kwargs: + raise ValueError("return_transform is not supported; use `with_preprocessor` instead") + + # Load the model and optionally preprocessing transforms + create_result = create_model_from_pretrained( + variant, return_transform=with_preprocessor, **kwargs) + model = create_result if not with_preprocessor else create_result[0] + + # Prepare for inference/model exporting purposes + model = reparameterize_model(model.visual.eval()) + + # Create example input for the model + image_size = model.image_size + pixel_values = torch.randn(1, 3, image_size[0], image_size[1], dtype=torch.float32) + example_input = (pixel_values,) + input_names = ["pixel_values"] + + # Create a wrapper module for the image encoder + class OpenCLIPImageEncoderWrapper(torch.nn.Module): + def __init__(self, base): + super().__init__() + self.base = base + def forward(self, pixel_values): + return torch.nn.functional.normalize(self.base(pixel_values), dim=-1) + wrapped_model = OpenCLIPImageEncoderWrapper(model) + + if with_preprocessor: + # Create a VisionPreprocessor for the model + preprocess = create_result[1] + vision_preprocessor = make_vision_preprocessor(preprocess) + return wrapped_model, example_input, input_names, vision_preprocessor + + return wrapped_model, example_input, input_names diff --git a/pyproject.toml b/pyproject.toml index 9d83c01..b3c0317 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ dev = [ "sphinx-copybutton>=0.5.0", "mermaidx>=0.9.0", "laya>=0.3.4", + "open_clip_torch>=3.3.0", ] [project.scripts] diff --git a/python/ggmlc/codegen/cpp.py b/python/ggmlc/codegen/cpp.py index 853a520..9b64f3c 100644 --- a/python/ggmlc/codegen/cpp.py +++ b/python/ggmlc/codegen/cpp.py @@ -58,6 +58,7 @@ def generate_header(self) -> str: "#include ", "#include ", "#include ", + "#include ", "#include ", "#include ", "#include ", @@ -78,7 +79,7 @@ def generate_header(self) -> str: f"namespace {self.model_name} {{", "", "// ----------------------------------------------------------------------------", - "// Broadcast helper for elementwise ops (mirrors the interpreter).", + "// Broadcast helper for elementwise ops.", "// ggml binary ops need `b` repeatable to `a`; this aligns the pair first.", "// ----------------------------------------------------------------------------", "inline std::pair match_broadcast(", @@ -117,10 +118,66 @@ def generate_header(self) -> str: " if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);", " b = ggml_repeat_4d(ctx, b, target_ne[0], target_ne[1], target_ne[2], target_ne[3]);", " }", + " if (a->type != b->type) {", + " if (a->type == GGML_TYPE_I32 && b->type == GGML_TYPE_F32) {", + " if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);", + " a = ggml_cast(ctx, a, GGML_TYPE_F32);", + " } else if (b->type == GGML_TYPE_I32 && a->type == GGML_TYPE_F32) {", + " if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);", + " b = ggml_cast(ctx, b, GGML_TYPE_F32);", + " }", + " }", " return {a, b};", "}", "", "// ----------------------------------------------------------------------------", + "// Pairwise concat helper.", + "// Skips empty inputs and aligns contiguity before concatenating.", + "// ----------------------------------------------------------------------------", + "inline struct ggml_tensor* concat_pair(", + " struct ggml_context* ctx,", + " struct ggml_tensor* a,", + " struct ggml_tensor* b,", + " int ggml_dim) {", + " auto is_empty = [](struct ggml_tensor* t) {", + " return t->ne[0] == 0 || t->ne[1] == 0 || t->ne[2] == 0 || t->ne[3] == 0;", + " };", + " if (is_empty(a)) return b;", + " if (is_empty(b)) return a;", + " if (!ggml_is_contiguous(a)) a = ggml_cont(ctx, a);", + " if (!ggml_is_contiguous(b)) b = ggml_cont(ctx, b);", + " return ggml_concat(ctx, a, b, ggml_dim);", + "}", + "", + "// ----------------------------------------------------------------------------", + "// Depthwise-conv layout helper.", + "// _direct wants weight [KW,KH,1,C] x input [W,H,C,N]; fold 1D", + "// layouts and cast to F32 (CUDA dw is F32-only). Mirrors the", + "// interpreter.", + "// ----------------------------------------------------------------------------", + "inline std::pair match_dw_layout(", + " struct ggml_context* ctx,", + " struct ggml_tensor* w,", + " struct ggml_tensor* x,", + " int is_1d) {", + " if (is_1d && w->ne[3] == 1) {", + " if (!ggml_is_contiguous(w)) w = ggml_cont(ctx, w);", + " w = ggml_cont(ctx, ggml_reshape_4d(ctx, w, w->ne[0], 1, w->ne[1], w->ne[2]));", + " }", + " if (is_1d && x->ne[3] == 1) {", + " if (!ggml_is_contiguous(x)) x = ggml_cont(ctx, x);", + " x = ggml_cont(ctx, ggml_reshape_4d(ctx, x, x->ne[0], 1, x->ne[1], x->ne[2]));", + " }", + " if (w->ne[2] != 1 && w->ne[3] == 1) {", + " if (!ggml_is_contiguous(w)) w = ggml_cont(ctx, w);", + " w = ggml_cont(ctx, ggml_reshape_4d(ctx, w, w->ne[0], w->ne[1], 1, w->ne[2]));", + " }", + " if (w->type != GGML_TYPE_F32) w = ggml_cast(ctx, w, GGML_TYPE_F32);", + " if (x->type != GGML_TYPE_F32) x = ggml_cast(ctx, x, GGML_TYPE_F32);", + " return {w, x};", + "}", + "", + "// ----------------------------------------------------------------------------", "// Model Parameter Weights Structure", "// ----------------------------------------------------------------------------", "struct Weights {", @@ -207,7 +264,10 @@ def generate_header(self) -> str: " const std::unordered_map& inputs,", " const std::unordered_map& symbols = {}", ") {", - " struct ggml_cgraph* gf = ggml_new_graph(ctx);", + # Each graph op expands to several ggml kernels, so size the + # arena generously from the node count instead of ggml's default. + f" size_t graph_nodes = std::max(32768, {len(self.graph.nodes) * 16});", + " struct ggml_cgraph* gf = ggml_new_graph_custom(ctx, graph_nodes, false);", "", " // Mapping from tensor ID to allocated computation node", " std::unordered_map tensors;", @@ -331,7 +391,11 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: elif node.opcode == GGMLOpCode.GGML_OP_UNARY: u_type = node.attributes.get("unary_op", "gelu") u_str = str(u_type).lower() - if "gelu" in u_str or u_type == 8: + if "erf" in u_str or u_type == 16: + lines.append( + f" tensors[{out_id}] = ggml_unary(ctx, {inp_vars[0]}, GGML_UNARY_OP_GELU_ERF);" + ) + elif "gelu" in u_str or u_type == 8: lines.append( f" tensors[{out_id}] = ggml_unary(ctx, {inp_vars[0]}, GGML_UNARY_OP_GELU);" ) @@ -376,10 +440,16 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: f" tensors[{out_id}] = ggml_unary(ctx, {inp_vars[0]}, GGML_UNARY_OP_RELU);" ) elif node.opcode == GGMLOpCode.GGML_OP_CLAMP: - min_v = node.attributes.get("min", 0.0) - max_v = node.attributes.get("max", 6.0) + if "min" in node.attributes: + min_expr = f"{node.attributes['min']}f" + else: + min_expr = "-FLT_MAX" + if "max" in node.attributes: + max_expr = f"{node.attributes['max']}f" + else: + max_expr = "FLT_MAX" lines.append( - f" tensors[{out_id}] = ggml_clamp(ctx, {inp_vars[0]}, {min_v}f, {max_v}f);" + f" tensors[{out_id}] = ggml_clamp(ctx, {inp_vars[0]}, {min_expr}, {max_expr});" ) elif node.opcode == GGMLOpCode.GGML_OP_CONV_2D: s0 = node.attributes.get("stride_w", 1) @@ -393,8 +463,13 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: ) if len(inp_vars) > 2: lines.append( - f" tensors[{out_id}] = ggml_add(ctx, tensors[{out_id}], {inp_vars[2]});" + f" auto bcb_{node.id} = match_broadcast(ctx, tensors[{out_id}], {inp_vars[2]});" + ) + lines.append( + f" tensors[{out_id}] = ggml_add(ctx, bcb_{node.id}.first, bcb_{node.id}.second);" ) + if node.attributes.get("fused_relu", 0): + lines.append(f" tensors[{out_id}] = ggml_relu(ctx, tensors[{out_id}]);") elif node.opcode == GGMLOpCode.GGML_OP_CONV_2D_DW: s0 = node.attributes.get("stride_w", 1) s1 = node.attributes.get("stride_h", 1) @@ -402,13 +477,22 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: p1 = node.attributes.get("pad_h", 0) d0 = node.attributes.get("dilation_w", 1) d1 = node.attributes.get("dilation_h", 1) + is_1d = int(node.attributes.get("is_1d", 0)) + lines.append( + f" auto dw_{node.id} = match_dw_layout(ctx, {inp_vars[0]}, {inp_vars[1]}, {is_1d});" + ) lines.append( - f" tensors[{out_id}] = ggml_conv_2d_dw(ctx, {inp_vars[0]}, {inp_vars[1]}, {s0}, {s1}, {p0}, {p1}, {d0}, {d1});" + f" tensors[{out_id}] = ggml_conv_2d_dw_direct(ctx, dw_{node.id}.first, dw_{node.id}.second, {s0}, {s1}, {p0}, {p1}, {d0}, {d1});" ) if len(inp_vars) > 2: lines.append( - f" tensors[{out_id}] = ggml_add(ctx, tensors[{out_id}], {inp_vars[2]});" + f" auto bcb_{node.id} = match_broadcast(ctx, tensors[{out_id}], {inp_vars[2]});" ) + lines.append( + f" tensors[{out_id}] = ggml_add(ctx, bcb_{node.id}.first, bcb_{node.id}.second);" + ) + if node.attributes.get("fused_relu", 0): + lines.append(f" tensors[{out_id}] = ggml_relu(ctx, tensors[{out_id}]);") elif node.opcode == GGMLOpCode.GGML_OP_POOL_2D: is_max = node.attributes.get("is_max", 0) != 0 pool_enum = "GGML_OP_POOL_MAX" if is_max else "GGML_OP_POOL_AVG" @@ -488,7 +572,12 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: k_var = inp_vars[1] v_var = inp_vars[2] if len(inp_vars) > 2 else "nullptr" mask_var = inp_vars[3] if len(inp_vars) > 3 else "nullptr" - scale = node.attributes.get("scale", 1.0) + # Absent scale means the PyTorch default 1/sqrt(head_dim), + # computed from q at runtime since the dim may be symbolic. + if "scale" in node.attributes: + scale_expr = f"{node.attributes['scale']}f" + else: + scale_expr = f"(1.0f / sqrtf((float)q_{node.id}->ne[0]))" lines.append( f" struct ggml_tensor* q_{node.id} = {q_var}; if (!ggml_is_contiguous(q_{node.id})) q_{node.id} = ggml_cont(ctx, q_{node.id});" ) @@ -499,8 +588,14 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: f" struct ggml_tensor* v_{node.id} = {v_var}; if (v_{node.id} && !ggml_is_contiguous(v_{node.id})) v_{node.id} = ggml_cont(ctx, v_{node.id});" ) lines.append( - f" tensors[{out_id}] = ggml_flash_attn_ext(ctx, q_{node.id}, k_{node.id}, v_{node.id}, {mask_var}, {scale}f, 0.0f, 0.0f);" + f" tensors[{out_id}] = ggml_flash_attn_ext(ctx, q_{node.id}, k_{node.id}, v_{node.id}, {mask_var}, {scale_expr}, 0.0f, 0.0f);" ) + # Raw output is heads-major; the graph layout needs L/H swapped + # unless the transpose is already fused upstream. + if not node.attributes.get("fused_transpose", 0): + lines.append( + f" tensors[{out_id}] = ggml_permute(ctx, tensors[{out_id}], 0, 2, 1, 3);" + ) elif node.opcode == GGMLOpCode.GGML_OP_ROPE: n_dims = node.attributes.get("n_dims", 0) mode = node.attributes.get("mode", 0) @@ -528,9 +623,27 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: ) elif node.opcode == GGMLOpCode.GGML_OP_CONCAT: dim = node.attributes.get("ggml_dim", node.attributes.get("dim", 0)) + if len(inp_vars) > 2: + parts = ", ".join(inp_vars) + lines.append(f" struct ggml_tensor* cc_parts_{node.id}[] = {{{parts}}};") + lines.append(f" struct ggml_tensor* cc_{node.id} = cc_parts_{node.id}[0];") + lines.append( + f" for (int ci_{node.id} = 1; ci_{node.id} < {len(inp_vars)}; ++ci_{node.id}) {{" + ) + lines.append( + f" cc_{node.id} = concat_pair(ctx, cc_{node.id}, cc_parts_{node.id}[ci_{node.id}], {dim});" + ) + lines.append(" }") + else: + lines.append(f" struct ggml_tensor* cc_{node.id} = {inp_vars[0]};") + if len(inp_vars) > 1: + lines.append( + f" cc_{node.id} = concat_pair(ctx, cc_{node.id}, {inp_vars[1]}, {dim});" + ) lines.append( - f" tensors[{out_id}] = ggml_concat(ctx, {inp_vars[0]}, {inp_vars[1]}, {dim});" + f" if (cc_{node.id} && !ggml_is_contiguous(cc_{node.id})) cc_{node.id} = ggml_cont(ctx, cc_{node.id});" ) + lines.append(f" tensors[{out_id}] = cc_{node.id};") elif node.opcode == GGMLOpCode.GGML_OP_CONT: lines.append(f" tensors[{out_id}] = ggml_cont(ctx, {inp_vars[0]});") elif node.opcode == GGMLOpCode.GGML_OP_SCALE: @@ -561,20 +674,19 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));" ) elif g_dim >= 2: + # Single-axis reduction: swap axis g_dim to position 0, + # reduce, swap back (ggml only reduces rows). + swap = "2, 1, 0, 3" if g_dim == 2 else "3, 1, 2, 0" lines.append(f" struct ggml_tensor* sr_{node.id} = {inp_vars[0]};") lines.append( f" if (!ggml_is_contiguous(sr_{node.id})) sr_{node.id} = ggml_cont(ctx, sr_{node.id});" ) lines.append( - f" sr_{node.id} = ggml_reshape_4d(ctx, sr_{node.id}, sr_{node.id}->ne[0], sr_{node.id}->ne[1] * sr_{node.id}->ne[2], 1, sr_{node.id}->ne[3]);" - ) - lines.append(f" sr_{node.id} = ggml_cont(ctx, sr_{node.id});") - lines.append( - f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));" + f" sr_{node.id} = ggml_cont(ctx, ggml_permute(ctx, sr_{node.id}, {swap}));" ) lines.append(f" sr_{node.id} = ggml_sum_rows(ctx, sr_{node.id});") lines.append( - f" sr_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, sr_{node.id}));" + f" sr_{node.id} = ggml_cont(ctx, ggml_permute(ctx, sr_{node.id}, {swap}));" ) else: lines.append(f" struct ggml_tensor* sr_{node.id} = {inp_vars[0]};") @@ -588,6 +700,45 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: lines.append( f" tensors[{out_id}] = ggml_reshape_4d(ctx, sr_{node.id}, {', '.join(ne_strs)});" ) + elif node.opcode == GGMLOpCode.GGML_OP_MEAN: + g_dim = node.attributes.get("ggml_dim", 0) + out_t = self.graph.tensors[out_id] + ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] + if g_dim == 1: + lines.append( + f" struct ggml_tensor* mn_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, {inp_vars[0]}));" + ) + lines.append(f" mn_{node.id} = ggml_mean(ctx, mn_{node.id});") + lines.append( + f" mn_{node.id} = ggml_cont(ctx, ggml_transpose(ctx, mn_{node.id}));" + ) + elif g_dim >= 2: + # Single-axis reduction: swap axis g_dim to position 0, + # reduce, swap back (ggml only reduces rows). + swap = "2, 1, 0, 3" if g_dim == 2 else "3, 1, 2, 0" + lines.append(f" struct ggml_tensor* mn_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(mn_{node.id})) mn_{node.id} = ggml_cont(ctx, mn_{node.id});" + ) + lines.append( + f" mn_{node.id} = ggml_cont(ctx, ggml_permute(ctx, mn_{node.id}, {swap}));" + ) + lines.append(f" mn_{node.id} = ggml_mean(ctx, mn_{node.id});") + lines.append( + f" mn_{node.id} = ggml_cont(ctx, ggml_permute(ctx, mn_{node.id}, {swap}));" + ) + else: + lines.append(f" struct ggml_tensor* mn_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(mn_{node.id})) mn_{node.id} = ggml_cont(ctx, mn_{node.id});" + ) + lines.append(f" mn_{node.id} = ggml_mean(ctx, mn_{node.id});") + lines.append( + f" if (!ggml_is_contiguous(mn_{node.id})) mn_{node.id} = ggml_cont(ctx, mn_{node.id});" + ) + lines.append( + f" tensors[{out_id}] = ggml_reshape_4d(ctx, mn_{node.id}, {', '.join(ne_strs)});" + ) elif node.opcode == GGMLOpCode.GGML_OP_CPY: out_t = self.graph.tensors[out_id] ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] @@ -607,7 +758,11 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] start = node.attributes.get("start", 0) ggml_dim = node.attributes.get("ggml_dim", 0) - offset = f"{start} * {inp_vars[0]}->nb[{ggml_dim}]" + mult = node.attributes.get("offset_mult", 1) + if mult == 1: + offset = f"{start} * {inp_vars[0]}->nb[{ggml_dim}]" + else: + offset = f"{start} * {mult} * {inp_vars[0]}->nb[{ggml_dim}]" lines.append( f" tensors[{out_id}] = ggml_view_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)}, {inp_vars[0]}->nb[1], {inp_vars[0]}->nb[2], {inp_vars[0]}->nb[3], {offset});" ) @@ -668,8 +823,11 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: ) lines.append(" #endif") else: - lines.append(f" // Generic fallback for opcode {node.opcode.name}") - lines.append(f" tensors[{out_id}] = {inp_vars[0]};") + raise NotImplementedError( + f"no C++ emission for {node.opcode.name} " + f"(node {node.id}, '{node.name}'); " + f"lower it to supported ops or add an emitter" + ) return lines diff --git a/python/ggmlc/dialect/ggml/lowering.py b/python/ggmlc/dialect/ggml/lowering.py index b7d5b16..f61f039 100644 --- a/python/ggmlc/dialect/ggml/lowering.py +++ b/python/ggmlc/dialect/ggml/lowering.py @@ -277,6 +277,11 @@ def _lower_op( in_t = c_graph.get_tensor(in_ids[0]) R = len(in_t.shape.dims) dim = attrs.get("dim", -1) + if isinstance(dim, (tuple, list)): + raise NotImplementedError( + f"Multi-axis MEAN '{op.name}' must be chained into single-axis " + f"ops by the frontend, got dim={dim}" + ) if dim < 0: dim += R ggml_dim = R - 1 - dim @@ -287,6 +292,11 @@ def _lower_op( R = len(in_t.shape.dims) axes = attrs.get("axes", None) or attrs.get("dim", -1) if isinstance(axes, (tuple, list)): + if len(axes) > 1: + raise NotImplementedError( + f"Multi-axis {opcode.value} '{op.name}' must be chained into " + f"single-axis ops by the frontend, got axes={axes}" + ) dim = axes[0] if len(axes) > 0 else -1 else: dim = int(axes) @@ -313,12 +323,18 @@ def _lower_op( op.name, ) elif opcode == OpCode.GELU: + approx = str(attrs.get("approximate", "none")).lower() + gelu_op = ( + GGMLUnaryOpCode.GGML_UNARY_OP_GELU_ERF + if approx in ("none", "erf") + else GGMLUnaryOpCode.GGML_UNARY_OP_GELU + ) return GGMLOpDef( op.id, GGMLOpCode.GGML_OP_UNARY, in_ids, out_ids, - {"unary_op": int(GGMLUnaryOpCode.GGML_UNARY_OP_GELU)}, + {"unary_op": int(gelu_op)}, op.name, ) elif opcode == OpCode.SILU: @@ -514,7 +530,21 @@ def _lower_op( ): w_ic = w_t.shape.dims[1].evaluate({}) x_ic = x_t.shape.dims[1].evaluate({}) - if w_ic == 1 and (groups > 1 or x_ic > 1): + w_oc = w_t.shape.dims[0].evaluate({}) if w_t.shape.dims[0].is_static() else None + if w_ic == 1 and groups > 1: + # True depthwise needs one filter per input channel. + # Grouped-with-multiplier (e.g. groups=80, C_in=80, + # C_out=160) has no ggml op and must be decomposed + # per group by the importer. + if (w_oc is None or w_oc == groups) and x_ic == groups: + is_dw = True + else: + raise ValueError( + f"grouped convolution '{op.name}' (id={op.id}) is not " + f"depthwise (groups={groups}, in_channels={x_ic}, " + f"out_channels={w_oc}); ggml has no grouped-conv op" + ) + elif w_ic == 1 and x_ic > 1: is_dw = True elif len(w_t.shape.dims) == 3 and w_t.shape.dims[1].is_static(): # Conv1d depthwise: weight [C, 1, K] @@ -568,6 +598,20 @@ def _lower_op( if dim < 0: dim += R ggml_dim = R - 1 - dim if R > 0 else 0 + if ggml_dim > 3: + # Outer dims fold into ne[3] (canonical_shape_to_ggml_ne): the + # slice lands on ggml dim 3, with an offset multiplier covering + # the folded dims inner to the sliced one. + ggml_dim = 3 + mult = 1 + for d in in_t.shape.dims[dim + 1 : R - 3]: + if isinstance(d, StaticDim): + mult *= d.value + else: + raise ValueError( # noqa: TRY004 - unsupported shape, not a type error + f"slice of folded dim {dim} needs static inner shape (tensor '{in_t.name}')" + ) + attrs["offset_mult"] = int(mult) attrs["ggml_dim"] = int(ggml_dim) attrs["start"] = int(start) return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_VIEW, in_ids, out_ids, attrs, op.name) @@ -592,7 +636,9 @@ def _lower_op( else: axes[i] = i if any(ax >= 4 for ax in axes): - return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_RESHAPE, in_ids, out_ids, attrs, op.name) + raise NotImplementedError( + f"Permute {op.name} results in a non-expressible ggml permute: axes={axes}" + ) attrs["axis0"] = axes[0] attrs["axis1"] = axes[1] attrs["axis2"] = axes[2] diff --git a/python/ggmlc/dialect/ggml/ops.py b/python/ggmlc/dialect/ggml/ops.py index b7db2f0..ca5e5cf 100644 --- a/python/ggmlc/dialect/ggml/ops.py +++ b/python/ggmlc/dialect/ggml/ops.py @@ -92,6 +92,7 @@ class GGMLUnaryOpCode(IntEnum): GGML_UNARY_OP_HARDSWISH = 11 GGML_UNARY_OP_HARDSIGMOID = 12 GGML_UNARY_OP_EXP = 13 + GGML_UNARY_OP_GELU_ERF = 16 @unique diff --git a/python/ggmlc/frontend/jax/importer.py b/python/ggmlc/frontend/jax/importer.py index eec9fc8..045460a 100644 --- a/python/ggmlc/frontend/jax/importer.py +++ b/python/ggmlc/frontend/jax/importer.py @@ -6,7 +6,7 @@ import numpy as np from ggmlc.ir.dtype import DType -from ggmlc.ir.graph import Graph +from ggmlc.ir.graph import Graph, chain_single_axis_reduction from ggmlc.ir.op import OpCode from ggmlc.ir.shape import Shape, StaticDim from ggmlc.ir.tensor import StorageClass, Tensor @@ -66,6 +66,58 @@ def _jax_dtype_to_dtype(dtype: Any) -> DType: return DType.from_numpy(dt) +def _pad_nhwc_asymmetric( + g: Graph, + t: Tensor, + h_pair: tuple[int, int], + w_pair: tuple[int, int], +) -> Tensor: + """Zero-pad NHWC sides ggml conv padding cannot express (top/left/bottom/right). + + Implemented as zero concats (fully supported in both runners); the caller + then uses the symmetric minimum as the conv padding. + """ + if t.dtype != DType.F32: + raise NotImplementedError(f"asymmetric pad needs F32 input, got {t.dtype}") + # NOTE: keep names free of brackets/colons: parameter names are matched + # verbatim by the GGUF loader, which chokes on jaxpr-style characters. + tag = f"padconv_{len(g.tensors)}" + dims = [int(d.value) if isinstance(d, StaticDim) else int(d) for d in t.shape.dims] + for axis, (before, after) in ((1, h_pair), (2, w_pair)): + for side, k in (("pre", before), ("post", after)): + if k <= 0: + continue + z_dims = list(dims) + z_dims[axis] = k + z = g.add_tensor( + name=f"{tag}_zeros_d{axis}_{side}", + shape=Shape.from_tuple(tuple(z_dims)), + dtype=t.dtype, + storage=StorageClass.CONSTANT, + data=np.zeros(tuple(z_dims), dtype=np.float32), + ) + g.parameters.append(z.id) + padded_dims = list(dims) + padded_dims[axis] = dims[axis] + k + out = g.add_tensor( + name=f"{tag}_pad_d{axis}_{side}", + shape=Shape.from_tuple(tuple(padded_dims)), + dtype=t.dtype, + storage=StorageClass.ACTIVATION, + ) + ins = [z.id, t.id] if side == "pre" else [t.id, z.id] + g.add_op( + opcode=OpCode.CONCAT, + inputs=ins, + outputs=[out.id], + attributes={"dim": axis}, + name=f"{tag}_pad_d{axis}_{side}", + ) + t = out + dims = padded_dims + return t + + def _import_equations( eqns: Sequence[Any], g: Graph, @@ -908,6 +960,21 @@ def _dim_val(d): else: lhs_t = g.get_tensor(in_tids[0]) + # Asymmetric padding (TF 'SAME' with odd totals, explicit pads): + # ggml convs only take symmetric padding, so materialize the + # excess sides as zero concats and keep the symmetric minimum. + pt0, pt1 = int(padding[0][0]), int(padding[0][1]) + pl0, pl1 = int(padding[1][0]), int(padding[1][1]) + sym_h, sym_w = min(pt0, pt1), min(pl0, pl1) + if (pt0, pt1) != (sym_h, sym_h) or (pl0, pl1) != (sym_w, sym_w): + lhs_t = _pad_nhwc_asymmetric( + g, + lhs_t, + (pt0 - sym_h, pt1 - sym_h), + (pl0 - sym_w, pl1 - sym_w), + ) + sym_padding = (sym_h, sym_w) + rhs_t = g.get_tensor(in_tids[1]) def _dim_val(d): @@ -972,7 +1039,7 @@ def _dim_val(d): ) conv_attrs = { "stride": tuple(window_strides), - "padding": (padding[0][0], padding[1][0]), + "padding": sym_padding, "dilation": tuple(rhs_dilation), "groups": int(feature_group_count), } @@ -994,7 +1061,7 @@ def _dim_val(d): else: conv_attrs = { "stride": tuple(window_strides), - "padding": (padding[0][0], padding[1][0]), + "padding": sym_padding, "dilation": tuple(rhs_dilation), "groups": int(feature_group_count), } @@ -1944,6 +2011,30 @@ def _dim_val(d): f"Unhandled primitive '{prim_name}' reached op emission with None opcode." ) + if opcode in (OpCode.SUM, OpCode.AMAX, OpCode.AMIN): + # Multi-axis reductions (e.g. global average pool) chain into + # single-axis ops; ggml reduces one axis (JAX out drops the axes). + raw_axes = eqn.params.get("axes", (-1,)) + if isinstance(raw_axes, (tuple, list)): + axes_list = [int(a) for a in raw_axes] + else: + axes_list = [int(raw_axes)] + in_t = g.get_tensor(in_tids[0]) + rank = len(in_t.shape.dims) + normed = [a + rank if a < 0 else a for a in axes_list] + if len(normed) > 1: + chained = chain_single_axis_reduction( + g, + in_t, + opcode, + normed, + False, + f"{prim_name}_{out_var}", + final_out=out_t, + ) + var_to_tensor[out_var] = chained + continue + g.add_op( opcode=opcode, inputs=in_tids, diff --git a/python/ggmlc/frontend/pytorch/importer.py b/python/ggmlc/frontend/pytorch/importer.py index 42f9c4c..dba6ca8 100644 --- a/python/ggmlc/frontend/pytorch/importer.py +++ b/python/ggmlc/frontend/pytorch/importer.py @@ -10,7 +10,7 @@ from ggmlc.frontend.pytorch.operators import get_opcode_for_aten from ggmlc.ir.dtype import DType -from ggmlc.ir.graph import Graph +from ggmlc.ir.graph import Graph, chain_single_axis_reduction from ggmlc.ir.op import OpCode from ggmlc.ir.shape import ( AddDim, @@ -81,6 +81,17 @@ def _torch_shape_to_shape(shape: Any) -> Shape: return Shape(dims) +def _norm_axes(dim: Any, rank: int) -> list[int]: + """Normalize a dim spec (int, list, tuple, or None) to non-negative axes.""" + if dim is None: + return [] + if isinstance(dim, (list, tuple)): + dims = [int(d) for d in dim] + else: + dims = [int(dim)] + return [d + rank if d < 0 else d for d in dims] + + def _restore_symbolic_batch_dim(g: Graph) -> None: """Copy a symbolic batch axis onto outputs that torch.export froze to the example B. @@ -253,6 +264,7 @@ def import_exported_program(ep: ExportedProgram, graph_name: str = "main") -> Gr or "sym_numel" in target_str or "chunk" in target_str or "split" in target_str + or "unbind" in target_str ): # Symbolic scalar query node or multi-output container (handled via getitem) continue @@ -261,16 +273,13 @@ def import_exported_program(ep: ExportedProgram, graph_name: str = "main") -> Gr parent = node.args[0] if isinstance(parent, Node): parent_target_str = str(parent.target) - if ("split" in parent_target_str or "chunk" in parent_target_str) and isinstance( - node.args[1], int - ): + if ( + "split" in parent_target_str + or "chunk" in parent_target_str + or "unbind" in parent_target_str + ) and isinstance(node.args[1], int): idx = int(node.args[1]) split_input = parent.args[0] - dim = ( - int(parent.args[2]) - if len(parent.args) > 2 and parent.args[2] is not None - else 0 - ) val = node.meta.get("val") shape = ( _torch_shape_to_shape(val.shape) @@ -280,28 +289,39 @@ def import_exported_program(ep: ExportedProgram, graph_name: str = "main") -> Gr dtype = ( DType.from_torch(val.dtype) if isinstance(val, torch.Tensor) else DType.F32 ) - if isinstance(val, torch.Tensor) and dim < 0: - dim = len(val.shape) + dim - if "chunk" in parent_target_str: - sz = val.shape[dim] if isinstance(val, torch.Tensor) else 1 - start = idx * sz - end = start + sz - elif isinstance(parent.args[1], (list, tuple)): - start = sum(parent.args[1][:idx]) - end = start + parent.args[1][idx] + + if "unbind" in parent_target_str: + dim = ( + int(parent.args[1]) + if len(parent.args) > 1 and parent.args[1] is not None + else 0 + ) + if isinstance(val, torch.Tensor) and dim < 0: + dim = len(val.shape) + dim + start = idx + end = idx + 1 + else: - sz = int(parent.args[1]) - start = idx * sz - end = (idx + 1) * sz - val = node.meta.get("val") - shape = ( - _torch_shape_to_shape(val.shape) - if isinstance(val, torch.Tensor) - else Shape([]) - ) - dtype = ( - DType.from_torch(val.dtype) if isinstance(val, torch.Tensor) else DType.F32 - ) + dim = ( + int(parent.args[2]) + if len(parent.args) > 2 and parent.args[2] is not None + else 0 + ) + if isinstance(val, torch.Tensor) and dim < 0: + dim = len(val.shape) + dim + + if "chunk" in parent_target_str: + sz = val.shape[dim] if isinstance(val, torch.Tensor) else 1 + start = idx * sz + end = start + sz + elif isinstance(parent.args[1], (list, tuple)): + start = sum(parent.args[1][:idx]) + end = start + parent.args[1][idx] + else: + sz = int(parent.args[1]) + start = idx * sz + end = (idx + 1) * sz + out_t = g.add_tensor( name=node.name, shape=shape, @@ -354,6 +374,7 @@ def import_exported_program(ep: ExportedProgram, graph_name: str = "main") -> Gr if ( "split" in target_str + or "unbind" in target_str or "assert" in target_str or "check" in target_str or "to.dtype" in target_str @@ -754,8 +775,70 @@ def _get_or_create_tensor(arg, default_name): attributes["dim1"] = 1 elif opcode == OpCode.PERMUTE: # aten.permute.default(self, dims) - input_tensor_ids.append(node_to_tensor[node.args[0]].id) - attributes["dims"] = [int(d) for d in node.args[1]] + in_t = node_to_tensor[node.args[0]] + p = [int(d) % len(in_t.shape.dims) for d in node.args[1]] + if len(in_t.shape.dims) == 5 and (len(p) == 0 or p[0] != 0): + # ggml only permutes 4D: squeeze static unit dims so the + # transpose becomes a plain 4D permute. The output stays + # squeezed (unit dims move no data); downstream selects + # index it via FX meta shapes, unaffected by the squeeze. + kept = [ + i + for i, d in enumerate(in_t.shape.dims) + if not (d.is_static() and d.evaluate({}) == 1) + ] + if len(kept) > 4: + raise NotImplementedError( + f"5D permute '{node.name}' with dims {p} is not " + f"expressible as a 4D ggml permute" + ) + sq_t = g.add_tensor( + name=f"{node.name}_sq5", + shape=Shape([in_t.shape.dims[i] for i in kept]), + dtype=in_t.dtype, + storage=StorageClass.ACTIVATION, + ) + g.add_op( + opcode=OpCode.RESHAPE, + inputs=[in_t.id], + outputs=[sq_t.id], + name=f"{node.name}_sq5", + ) + kept_out = [k for k in range(len(p)) if p[k] in kept] + p_reduced = [kept.index(p[k]) for k in kept_out] + val = node.meta.get("val") + out_shape = ( + _torch_shape_to_shape(val.shape) + if val is not None and isinstance(val, torch.Tensor) + else Shape([in_t.shape.dims[p[k]] for k in range(len(p))]) + ) + # Squeeze out unit dims so lowering sees a consistent 4D permute + if ( + len(in_t.shape.dims) == 5 + and out_shape.dims + and out_shape.dims.count(StaticDim(1)) > 0 + ): + out_shape = Shape( + d for d in out_shape.dims if not (d.is_static() and d.evaluate({}) == 1) + ) + out_t = g.add_tensor( + name=node.name, + shape=out_shape, + dtype=in_t.dtype, + storage=StorageClass.ACTIVATION, + ) + g.add_op( + opcode=OpCode.PERMUTE, + inputs=[sq_t.id], + outputs=[out_t.id], + attributes={"dims": p_reduced}, + name=node.name, + ) + node_to_tensor[node] = out_t + name_to_tensor[node.name] = out_t + continue + input_tensor_ids.append(in_t.id) + attributes["dims"] = p elif opcode == OpCode.SLICE: input_tensor_ids.append(node_to_tensor[node.args[0]].id) if "select" in str(node.target): @@ -777,18 +860,41 @@ def _get_or_create_tensor(arg, default_name): attributes["start"] = start attributes["end"] = end attributes["step"] = step - elif opcode == OpCode.MEAN: + elif opcode == OpCode.GELU: input_tensor_ids.append(node_to_tensor[node.args[0]].id) + approx = node.kwargs.get("approximate", node.args[1] if len(node.args) > 1 else "none") + attributes["approximate"] = str(approx).lower() + elif opcode == OpCode.MEAN: + in_t = node_to_tensor[node.args[0]] dim = node.args[1] if len(node.args) > 1 else -1 - attributes["dim"] = int(dim[0]) if isinstance(dim, (list, tuple)) else int(dim) - attributes["keepdim"] = 1 if len(node.args) > 2 and node.args[2] else 0 + keepdim = bool(len(node.args) > 2 and node.args[2]) + normed = _norm_axes(dim, len(in_t.shape.dims)) + if len(normed) > 1: + # Multi-dim reduction: chain single-dim means (ggml reduces + # one axis); keepdim intermediates keep ranks aligned. + cur_t = chain_single_axis_reduction( + g, in_t, OpCode.MEAN, normed, keepdim, node.name + ) + node_to_tensor[node] = cur_t + name_to_tensor[node.name] = cur_t + continue + input_tensor_ids.append(in_t.id) + attributes["dim"] = normed[0] if normed else -1 + attributes["keepdim"] = 1 if keepdim else 0 elif opcode in (OpCode.SUM, OpCode.AMAX, OpCode.AMIN): - input_tensor_ids.append(node_to_tensor[node.args[0]].id) + in_t = node_to_tensor[node.args[0]] dim = node.kwargs.get("dim", node.args[1] if len(node.args) > 1 else -1) - if isinstance(dim, (list, tuple)): - dim = dim[0] if len(dim) > 0 else -1 - attributes["dim"] = int(dim) if dim is not None else -1 keepdim = node.kwargs.get("keepdim", node.args[2] if len(node.args) > 2 else False) + normed = _norm_axes(dim, len(in_t.shape.dims)) + if len(normed) > 1: + cur_t = chain_single_axis_reduction( + g, in_t, opcode, normed, bool(keepdim), node.name + ) + node_to_tensor[node] = cur_t + name_to_tensor[node.name] = cur_t + continue + input_tensor_ids.append(in_t.id) + attributes["dim"] = normed[0] if normed else -1 attributes["keepdim"] = 1 if keepdim else 0 elif opcode == OpCode.CONCAT: # aten.cat.default(tensors, dim=0) @@ -797,15 +903,19 @@ def _get_or_create_tensor(arg, default_name): input_tensor_ids.append(node_to_tensor[sub_node].id) attributes["dim"] = int(node.args[1]) if len(node.args) > 1 else 0 elif opcode == OpCode.EXPAND: - # aten.expand.default(self, size) input_tensor_ids.append(node_to_tensor[node.args[0]].id) - dims = [] - for d in node.args[1]: - if isinstance(d, Node): - dims.append(_symint_to_dim(d.meta.get("val"))) - else: - dims.append(_symint_to_dim(d)) - attributes["shape"] = tuple(dims) + if "expand_as" in target_str: + # aten.expand_as.default(self, other) + attributes["shape"] = tuple(node_to_tensor[node.args[1]].shape.dims) + else: + # aten.expand.default(self, size) + dims = [] + for d in node.args[1]: + if isinstance(d, Node): + dims.append(_symint_to_dim(d.meta.get("val"))) + else: + dims.append(_symint_to_dim(d)) + attributes["shape"] = tuple(dims) elif opcode in (OpCode.SQUEEZE, OpCode.UNSQUEEZE): input_tensor_ids.append(node_to_tensor[node.args[0]].id) if len(node.args) > 1 and node.args[1] is not None: @@ -1060,7 +1170,10 @@ def _get_or_create_tensor(arg, default_name): else attributes["dilation_h"] ) attributes["groups"] = groups - # Check if this is a grouped convolution that needs decomposition (1 < groups < in_channels) + # Check if this is a grouped convolution that needs decomposition: + # 1 < groups < in_channels, or groups == in_channels with a channel + # multiplier (out_channels != in_channels, so not depthwise). + # True depthwise (groups == in == out) stays a single DW conv. in_t_candidate = node_to_tensor[node.args[0]] w_t_candidate = node_to_tensor[node.args[1]] bias_t_candidate = ( @@ -1081,7 +1194,14 @@ def _get_or_create_tensor(arg, default_name): cin_val = int(in_t_candidate.shape.dims[1].evaluate({})) cout_val = int(w_t_candidate.shape.dims[0].evaluate({})) g_val = int(groups) - if 1 < g_val < cin_val: + is_true_dw = g_val == cin_val == cout_val + if ( + g_val > 1 + and not is_true_dw + and g_val <= cin_val + and cin_val % g_val == 0 + and cout_val % g_val == 0 + ): cin_per_group = cin_val // g_val cout_per_group = cout_val // g_val out_group_tensors = [] diff --git a/python/ggmlc/frontend/pytorch/operators.py b/python/ggmlc/frontend/pytorch/operators.py index be5f36a..af047b3 100644 --- a/python/ggmlc/frontend/pytorch/operators.py +++ b/python/ggmlc/frontend/pytorch/operators.py @@ -87,8 +87,8 @@ "aten.numpy_T.default": OpCode.TRANSPOSE, "aten.slice.Tensor": OpCode.SLICE, "aten.cat.default": OpCode.CONCAT, - "aten.split.Tensor": OpCode.SPLIT, "aten.expand.default": OpCode.EXPAND, + "aten.expand_as.default": OpCode.EXPAND, "aten.squeeze.dim": OpCode.SQUEEZE, "aten.unsqueeze.default": OpCode.UNSQUEEZE, "aten.sum.dim_IntList": OpCode.SUM, diff --git a/python/ggmlc/ir/graph.py b/python/ggmlc/ir/graph.py index 99ab876..f05147b 100644 --- a/python/ggmlc/ir/graph.py +++ b/python/ggmlc/ir/graph.py @@ -5,7 +5,7 @@ from ggmlc.ir.dtype import DType from ggmlc.ir.op import OpCode, Operation -from ggmlc.ir.shape import Shape +from ggmlc.ir.shape import Shape, StaticDim from ggmlc.ir.state import StateDeclaration from ggmlc.ir.tensor import StorageClass, Tensor @@ -167,3 +167,51 @@ def visualize(self, output_path: str = "graph.html", format: str = "auto") -> Pa from ggmlc.visualization.mermaid import visualize return visualize(self, output_path=output_path, format=format, title=self.name) + + +def chain_single_axis_reduction( + g: Graph, + in_t: Tensor, + opcode: OpCode, + normed: list[int], + keepdim: bool, + base_name: str, + final_out: Tensor | None = None, +) -> Tensor: + """Chain a multi-axis reduction into single-axis ops (ggml reduces one axis). + + Intermediates keep rank (unit dims) so later indices stay valid; only the + last op honors keepdim (dropping every reduced dim when False). + """ + tag = opcode.value + cur_id = in_t.id + cur_dims = list(in_t.shape.dims) + cur_t = in_t + for j, d in enumerate(normed): + last = j == len(normed) - 1 + kd = keepdim if last else True + out_dims = list(cur_dims) + if kd: + out_dims[d] = StaticDim(1) + else: + doomed = set(normed) + out_dims = [dd for i, dd in enumerate(cur_dims) if i not in doomed] + if last and final_out is not None: + cur_t = final_out + else: + cur_t = g.add_tensor( + name=base_name if last else f"{base_name}_{tag}_d{d}", + shape=Shape(out_dims), + dtype=in_t.dtype, + storage=StorageClass.ACTIVATION, + ) + g.add_op( + opcode=opcode, + inputs=[cur_id], + outputs=[cur_t.id], + attributes={"dim": d, "keepdim": 1 if kd else 0}, + name=f"{base_name}_{tag}_d{d}", + ) + cur_id = cur_t.id + cur_dims = out_dims + return cur_t diff --git a/python/ggmlc/ir/op.py b/python/ggmlc/ir/op.py index f8794ae..453db32 100644 --- a/python/ggmlc/ir/op.py +++ b/python/ggmlc/ir/op.py @@ -61,7 +61,6 @@ class OpCode(Enum): TRANSPOSE = "transpose" SLICE = "slice" CONCAT = "concat" - SPLIT = "split" EXPAND = "expand" REPEAT = "repeat" SQUEEZE = "squeeze" diff --git a/python/ggmlc/serialization/gguf.py b/python/ggmlc/serialization/gguf.py index 7ef7386..1049fd8 100644 --- a/python/ggmlc/serialization/gguf.py +++ b/python/ggmlc/serialization/gguf.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import io import json import struct @@ -129,14 +130,14 @@ def _serialize_attr_val(v: Any) -> Any: return str(v) -def _graph_to_json_spec(graph: GGMLExecutionGraph) -> str: +def _graph_to_json_spec(graph: GGMLExecutionGraph, name_map: dict[int, str] | None = None) -> str: """Serializes the execution graph DAG structure into a JSON specification string.""" tensors_spec = {} for tid, t in graph.tensors.items(): storage_int = STORAGE_TO_INT.get(t.storage, 0) tensors_spec[str(tid)] = { "id": t.id, - "name": t.name, + "name": name_map.get(tid, t.name) if name_map else t.name, "type": int(t.ggml_type), "ne": [_dim_to_dict(d) for d in t.ne], "storage": storage_int, @@ -378,6 +379,31 @@ def write_to_file(self, filepath: str | Path) -> Path: return p +def _shorten_tensor_names(graph: GGMLExecutionGraph) -> dict[int, str]: + """Map tensor ids to GGUF-safe names (< GGML_MAX_NAME chars, unique). + + Short names pass through untouched (runner input names stay stable); + only overlong or colliding names are truncated with a hash suffix. + """ + name_map: dict[int, str] = {} + seen: set[str] = set() + for tid in sorted(graph.tensors.keys()): + name = graph.tensors[tid].name or f"tensor_{tid}" + if len(name) < 64 and name not in seen: + name_map[tid] = name + seen.add(name) + continue + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:8] + cand = f"{name[:54]}_{digest}" + n = 0 + while cand in seen: + n += 1 + cand = f"{name[:52]}_{digest}_{n}"[:63] + name_map[tid] = cand + seen.add(cand) + return name_map + + def _build_gguf_writer( graph: GGMLExecutionGraph, extra_metadata: dict[str, Any] | None = None, @@ -413,17 +439,20 @@ def _build_gguf_writer( writer.add_string_array("ggmlc.symbol_table", graph.symbol_table) - # 2. Graph Spec JSON Metadata - spec_json = _graph_to_json_spec(graph) + # 2. Graph Spec JSON Metadata (tensor names shortened consistently with + # the tensor-info table: ggml rejects names >= GGML_MAX_NAME (64), and + # the native loader binds data by name, so both sides share the mapping) + name_map = _shorten_tensor_names(graph) + spec_json = _graph_to_json_spec(graph, name_map) writer.add_string("ggmlc.graph_spec", spec_json) # 4. Add Tensors with data (Parameters / Constants) used_names: set[str] = set() for _tid, t in sorted(graph.tensors.items()): if t.data is not None: - name = t.name + name = name_map.get(_tid, t.name) if name in used_names: - name = f"{name}_{_tid}" + name = f"{name[:56]}_{_tid}"[:63] used_names.add(name) # Prepare raw byte buffer and match GGUF shape to concrete data diff --git a/runtime/src/executor.cpp b/runtime/src/executor.cpp index 7f5ef78..bb7ca2a 100644 --- a/runtime/src/executor.cpp +++ b/runtime/src/executor.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include "ggml.h" #include "ggml-impl.h" #include "ggml-backend.h" @@ -1891,12 +1892,19 @@ void ModelExecutor::prepare(const std::unordered_map& symb t = ggml_mean(ctx_, t); result = ggml_cont(ctx_, ggml_transpose(ctx_, t)); } else if (g_dim >= 2) { - // Spatial reduction over dims 1 and 2 (e.g. NHWC global pool: [C, W, H, B] -> [C, 1, 1, B]) - struct ggml_tensor* flat_hw = reshape4d_contig(ctx_, in0, in0->ne[0], in0->ne[1] * in0->ne[2], 1, in0->ne[3]); - flat_hw = ggml_cont(ctx_, flat_hw); - struct ggml_tensor* t = ggml_cont(ctx_, ggml_transpose(ctx_, flat_hw)); - t = ggml_mean(ctx_, t); - result = ggml_cont(ctx_, ggml_transpose(ctx_, t)); + // Single-axis reduction: swap axis g_dim to position 0, + // reduce, swap back (ggml only reduces rows). + struct ggml_tensor* t = in0; + if (!ggml_is_contiguous(t)) t = ggml_cont(ctx_, t); + if (g_dim == 2) { + t = ggml_cont(ctx_, ggml_permute(ctx_, t, 2, 1, 0, 3)); + t = ggml_mean(ctx_, t); + result = ggml_cont(ctx_, ggml_permute(ctx_, t, 2, 1, 0, 3)); + } else { + t = ggml_cont(ctx_, ggml_permute(ctx_, t, 3, 1, 2, 0)); + t = ggml_mean(ctx_, t); + result = ggml_cont(ctx_, ggml_permute(ctx_, t, 3, 1, 2, 0)); + } } else { result = ggml_mean(ctx_, in0); } @@ -1912,12 +1920,19 @@ void ModelExecutor::prepare(const std::unordered_map& symb t = ggml_sum_rows(ctx_, t); result = ggml_cont(ctx_, ggml_transpose(ctx_, t)); } else if (g_dim >= 2) { - // Spatial reduction over dims 1 and 2 (e.g. NHWC global pool: [C, W, H, B] -> [C, 1, 1, B]) - struct ggml_tensor* flat_hw = reshape4d_contig(ctx_, in0, in0->ne[0], in0->ne[1] * in0->ne[2], 1, in0->ne[3]); - flat_hw = ggml_cont(ctx_, flat_hw); - struct ggml_tensor* t = ggml_cont(ctx_, ggml_transpose(ctx_, flat_hw)); - t = ggml_sum_rows(ctx_, t); - result = ggml_cont(ctx_, ggml_transpose(ctx_, t)); + // Single-axis reduction: swap axis g_dim to position 0, + // reduce, swap back (ggml only reduces rows). + struct ggml_tensor* t = in0; + if (!ggml_is_contiguous(t)) t = ggml_cont(ctx_, t); + if (g_dim == 2) { + t = ggml_cont(ctx_, ggml_permute(ctx_, t, 2, 1, 0, 3)); + t = ggml_sum_rows(ctx_, t); + result = ggml_cont(ctx_, ggml_permute(ctx_, t, 2, 1, 0, 3)); + } else { + t = ggml_cont(ctx_, ggml_permute(ctx_, t, 3, 1, 2, 0)); + t = ggml_sum_rows(ctx_, t); + result = ggml_cont(ctx_, ggml_permute(ctx_, t, 3, 1, 2, 0)); + } } else { result = ggml_sum_rows(ctx_, in0); } @@ -1967,6 +1982,8 @@ void ModelExecutor::prepare(const std::unordered_map& symb result = ggml_sigmoid(ctx_, act_in); } else if (u == 8) { // GELU result = ggml_gelu(ctx_, act_in); + } else if (u == 16) { // GELU_ERF + result = ggml_unary(ctx_, act_in, GGML_UNARY_OP_GELU_ERF); } else if (u == 10) { // SILU result = ggml_silu(ctx_, act_in); } else if (u == 11) { // HARDSWISH @@ -2496,6 +2513,7 @@ void ModelExecutor::prepare(const std::unordered_map& symb int g_dim = op.attributes.count("ggml_dim") ? static_cast(op.attributes.at("ggml_dim")) : 0; int64_t start = op.attributes.count("start") ? op.attributes.at("start") : 0; int64_t step = op.attributes.count("step") ? op.attributes.at("step") : 1; + int64_t mult = op.attributes.count("offset_mult") ? op.attributes.at("offset_mult") : 1; // Default: keep QKV fusion slices as non-contiguous views (zero copy). // GGMLC_VIEW_FORCE_CONT=1 restores eager materialization for A/B. if (env_flag_enabled("GGMLC_VIEW_FORCE_CONT")) { @@ -2506,7 +2524,7 @@ void ModelExecutor::prepare(const std::unordered_map& symb // Break pathological view-of-view chains that break offset math. in0 = ggml_cont(ctx_, in0); } - size_t offset = start * in0->nb[g_dim]; + size_t offset = static_cast(start * mult) * in0->nb[g_dim]; size_t nb1 = in0->nb[1] * (g_dim == 1 ? step : 1); size_t nb2 = in0->nb[2] * (g_dim == 2 ? step : 1); size_t nb3 = in0->nb[3] * (g_dim == 3 ? step : 1); @@ -2621,8 +2639,7 @@ void ModelExecutor::prepare(const std::unordered_map& symb // CUDA conv2d-dw is F32-only. if (in0->type != GGML_TYPE_F32) in0 = ggml_cast(ctx_, in0, GGML_TYPE_F32); if (in1->type != GGML_TYPE_F32) in1 = ggml_cast(ctx_, in1, GGML_TYPE_F32); - result = is_1d ? ggml_conv_2d_dw_direct(ctx_, in0, in1, s0, s1, p0, p1, d0, d1) - : ggml_conv_2d_dw(ctx_, in0, in1, s0, s1, p0, p1, d0, d1); + result = ggml_conv_2d_dw_direct(ctx_, in0, in1, s0, s1, p0, p1, d0, d1); if (op.inputs.size() > 2) { struct ggml_tensor* bias = ggml_tensors_[op.inputs[2]]; if (bias) { @@ -2646,9 +2663,9 @@ void ModelExecutor::prepare(const std::unordered_map& symb } case GGML_OP_CLAMP: { float min_val = op.float_attributes.count("min") ? static_cast(op.float_attributes.at("min")) - : op.attributes.count("min") ? static_cast(op.attributes.at("min")) : 0.0f; + : op.attributes.count("min") ? static_cast(op.attributes.at("min")) : -FLT_MAX; float max_val = op.float_attributes.count("max") ? static_cast(op.float_attributes.at("max")) - : op.attributes.count("max") ? static_cast(op.attributes.at("max")) : 6.0f; + : op.attributes.count("max") ? static_cast(op.attributes.at("max")) : FLT_MAX; result = ggml_clamp(ctx_, in0, min_val, max_val); break; } diff --git a/tests/codegen/test_standalone_execution.py b/tests/codegen/test_standalone_execution.py index 8483e59..13be284 100644 --- a/tests/codegen/test_standalone_execution.py +++ b/tests/codegen/test_standalone_execution.py @@ -21,6 +21,7 @@ import torch from ggmlc.codegen import generate_cpp_project from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.dialect.ggml.ops import GGMLOpCode from ggmlc.frontend.pytorch import export_torch_model from ggmlc.serialization.gguf import save_to_gguf from torch import nn @@ -379,3 +380,235 @@ def forward(self, img_embeds): _run_standalone( L2Normalize().eval(), (img_embeds,), "tiny_l2_normalize", ggml_standalone_libs, tmp_path ) + + +def test_standalone_conv2d_bias(ggml_standalone_libs, tmp_path): + """CONV_2D with channel-vector bias through generated code matches torch.""" + torch.manual_seed(0) + + class ConvBias(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 8, 3, padding=1) + + def forward(self, x): + return self.conv(x) + + x = torch.randn(1, 3, 16, 16) + _run_standalone(ConvBias().eval(), (x,), "tiny_conv2d_bias", ggml_standalone_libs, tmp_path) + + +def test_standalone_concat_multi(ggml_standalone_libs, tmp_path): + """Multi-way CONCAT (beyond the first two inputs) matches torch.""" + torch.manual_seed(0) + + class ConcatFour(nn.Module): + def forward(self, a, b, c, d): + return torch.cat([a, b, c, d], dim=1) + + args = tuple(torch.randn(1, 2, 8, 8) for _ in range(4)) + _run_standalone(ConcatFour().eval(), args, "tiny_concat_multi", ggml_standalone_libs, tmp_path) + + +def test_standalone_depthwise_conv2d(ggml_standalone_libs, tmp_path): + """True DEPTHWISE conv (direct kernel) through generated code matches torch.""" + torch.manual_seed(0) + + class Depthwise(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(8, 8, 3, padding=1, groups=8) + + def forward(self, x): + return self.conv(x) + + x = torch.randn(1, 8, 16, 16) + _run_standalone( + Depthwise().eval(), (x,), "tiny_depthwise_conv2d", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_clamp_min_open(ggml_standalone_libs, tmp_path): + """Open-ended clamp_min (no max) must not invent a max bound.""" + torch.manual_seed(0) + + class ClampMin(nn.Module): + def forward(self, x): + return (x * 10.0).clamp_min(0.5) + + x = torch.randn(4, 16) + _run_standalone(ClampMin().eval(), (x,), "tiny_clamp_min_open", ggml_standalone_libs, tmp_path) + + +def test_standalone_fused_conv_relu(ggml_standalone_libs, tmp_path): + """Fused CONV_2D+RELU through generated code matches torch.""" + torch.manual_seed(0) + + class ConvRelu(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 8, 3, padding=1) + + def forward(self, x): + return torch.relu(self.conv(x)) + + model = ConvRelu().eval() + x = torch.randn(1, 3, 16, 16) + exported = export_torch_model(model, (x,), model_name="tiny_fused_conv_relu") + ggml_graph = lower_to_ggml(exported.main_graph) + fused = [ + op + for op in ggml_graph.nodes + if op.opcode == GGMLOpCode.GGML_OP_CONV_2D and op.attributes.get("fused_relu") + ] + assert fused, "expected conv+relu fusion for coverage" + _run_standalone(model, (x,), "tiny_fused_conv_relu", ggml_standalone_libs, tmp_path) + + +def test_standalone_sdpa_default_scale(ggml_standalone_libs, tmp_path): + """SDPA without explicit scale uses 1/sqrt(head_dim), matching torch.""" + torch.manual_seed(0) + + class TinyAttention(nn.Module): + def forward(self, q, k, v): + return torch.nn.functional.scaled_dot_product_attention(q, k, v) + + q = torch.randn(1, 2, 8, 16) + k = torch.randn(1, 2, 8, 16) + v = torch.randn(1, 2, 8, 16) + _run_standalone( + TinyAttention().eval(), (q, k, v), "tiny_sdpa_default_scale", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_mean_last_dim(ggml_standalone_libs, tmp_path): + """MEAN over the last dim through generated code matches torch.""" + torch.manual_seed(0) + + class MeanLast(nn.Module): + def forward(self, x): + return x.mean(dim=-1) + + x = torch.randn(4, 16) + _run_standalone(MeanLast().eval(), (x,), "tiny_mean_last_dim", ggml_standalone_libs, tmp_path) + + +def test_standalone_mean_middle_dim(ggml_standalone_libs, tmp_path): + """MEAN over a middle dim (transpose reduction path) matches torch.""" + torch.manual_seed(0) + + class MeanMiddle(nn.Module): + def forward(self, x): + return x.mean(dim=2, keepdim=True) + + x = torch.randn(1, 2, 4, 8) + _run_standalone( + MeanMiddle().eval(), (x,), "tiny_mean_middle_dim", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_mean_two_dims(ggml_standalone_libs, tmp_path): + """MEAN over two dims (chained decomposition) matches torch.""" + torch.manual_seed(0) + + class MeanTwoDims(nn.Module): + def forward(self, x): + return x.mean(dim=(2, 3), keepdim=True) + + x = torch.randn(1, 2, 4, 8) + _run_standalone( + MeanTwoDims().eval(), (x,), "tiny_mean_two_dims", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_gelu_exact(ggml_standalone_libs, tmp_path): + """Exact (erf) GELU through generated code matches torch.""" + torch.manual_seed(0) + + class GeluExact(nn.Module): + def forward(self, x): + return torch.nn.functional.gelu(x * 2.0) + + x = torch.randn(2, 16) + _run_standalone(GeluExact().eval(), (x,), "tiny_gelu_exact", ggml_standalone_libs, tmp_path) + + +def test_standalone_mean_channel_dim(ggml_standalone_libs, tmp_path): + """MEAN over torch dim 1 (axis-swap reduction path) matches torch.""" + torch.manual_seed(0) + + class MeanChannel(nn.Module): + def forward(self, x): + return x.mean(dim=1, keepdim=True) + + x = torch.randn(1, 4, 8, 8) + _run_standalone( + MeanChannel().eval(), (x,), "tiny_mean_channel_dim", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_mean_batch_dim(ggml_standalone_libs, tmp_path): + """MEAN over torch dim 0 (axis-swap reduction path) matches torch.""" + torch.manual_seed(0) + + class MeanBatch(nn.Module): + def forward(self, x): + return x.mean(dim=0, keepdim=True) + + x = torch.randn(2, 4, 8, 8) + _run_standalone(MeanBatch().eval(), (x,), "tiny_mean_batch_dim", ggml_standalone_libs, tmp_path) + + +def test_standalone_sum_channel_dim(ggml_standalone_libs, tmp_path): + """SUM over torch dim 1 (axis-swap reduction path) matches torch.""" + torch.manual_seed(0) + + class SumChannel(nn.Module): + def forward(self, x): + return x.sum(dim=1, keepdim=True) + + x = torch.randn(1, 4, 8, 8) + _run_standalone( + SumChannel().eval(), (x,), "tiny_sum_channel_dim", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_qkv_split_permute(ggml_standalone_libs, tmp_path): + """QKV reshape/5D-permute/unbind plumbing preserves head/slot order.""" + torch.manual_seed(0) + + class QKVSplit(nn.Module): + def __init__(self): + super().__init__() + self.proj = nn.Linear(32, 3 * 4 * 8, bias=False) + + def forward(self, x): + b, _c, h, w = x.shape + qkv = self.proj(x.flatten(2).transpose(1, 2)) + qkv = qkv.reshape(b, h * w, 3, 4, 8).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + return q + 2 * k + 3 * v + + x = torch.randn(1, 32, 4, 4) + _run_standalone( + QKVSplit().eval(), (x,), "tiny_qkv_split_permute", ggml_standalone_libs, tmp_path + ) + + +def test_standalone_gelu_tanh(ggml_standalone_libs, tmp_path): + """Tanh-approximate GELU through generated code matches torch (loose tol).""" + torch.manual_seed(0) + + class GeluTanh(nn.Module): + def forward(self, x): + return torch.nn.functional.gelu(x * 2.0, approximate="tanh") + + x = torch.randn(2, 16) + _run_standalone( + GeluTanh().eval(), + (x,), + "tiny_gelu_tanh", + ggml_standalone_libs, + tmp_path, + atol=5e-3, + ) diff --git a/tests/e2e/test_openclip_models.py b/tests/e2e/test_openclip_models.py new file mode 100644 index 0000000..534356c --- /dev/null +++ b/tests/e2e/test_openclip_models.py @@ -0,0 +1,45 @@ +"""End-to-end tests for OpenCLIP models.""" + +import torch +from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.frontend.pytorch import export_torch_model +from ggmlc.runtime.runner import ModelRunner +from ggmlc.serialization.graph import serialize_ggml_graph +from ggmlc.validation.numerical import check_numerical_accuracy + +from examples.models.openclip_model import load_openclip_image_encoder_model + + +def test_mobileclip2_s2_cpu(): + """Verifies MobileCLIP2-S2 on CPU backend.""" + + model_name = "MobileCLIP2-S2" + variant = "hf-hub:timm/MobileCLIP2-S2-OpenCLIP" + + [model, example_input, input_names] = load_openclip_image_encoder_model( + variant=variant, with_preprocessor=False + ) + + assert input_names == ["pixel_values"] + + with torch.inference_mode(): + ref = model(*example_input).numpy() + + exported = export_torch_model(model, example_input, model_name=model_name) + assert len(exported.main_graph.nodes) > 0 + + ggml_graph = lower_to_ggml(exported.main_graph) + ser_bytes = serialize_ggml_graph(ggml_graph) + + runner = ModelRunner(ser_bytes, device="cpu") + act = runner(*[x.numpy() for x in example_input]) + if isinstance(act, dict): + act = next(iter(act.values())) + act = act.reshape(ref.shape) + + result = check_numerical_accuracy(ref, act, atol=1e-3) + assert result.passed, f"{model_name} numerical parity failed: {result}" + + +if __name__ == "__main__": + test_mobileclip2_s2_cpu() diff --git a/tests/frontend/test_importer_decompositions.py b/tests/frontend/test_importer_decompositions.py new file mode 100644 index 0000000..7fbdbab --- /dev/null +++ b/tests/frontend/test_importer_decompositions.py @@ -0,0 +1,167 @@ +"""Structure-level tests for importer decompositions and lowering guards. + +Fast (no execution, no compilation): assert the graph shapes the +importer/lowering produce for patterns the branch fixes. +""" + +import pytest +import torch +from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.dialect.ggml.ops import GGMLOpCode +from ggmlc.frontend.pytorch import export_torch_model +from ggmlc.ir.dtype import DType +from ggmlc.ir.graph import Graph +from ggmlc.ir.op import OpCode +from ggmlc.ir.shape import Shape +from ggmlc.ir.tensor import StorageClass +from torch import nn + + +class QKVSplit(nn.Module): + def __init__(self): + super().__init__() + self.proj = nn.Linear(32, 3 * 4 * 8, bias=False) + + def forward(self, x): + b, _c, h, w = x.shape + qkv = self.proj(x.flatten(2).transpose(1, 2)) + qkv = qkv.reshape(b, h * w, 3, 4, 8).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + return q + 2 * k + 3 * v + + +def _exported(model, args): + return export_torch_model(model, args, model_name="t", enable_fusion=False) + + +def test_5d_permute_decomposes_to_4d(): + """Non-batch-first 5D permutes become squeeze + 4D permute (no 5D survives).""" + torch.manual_seed(0) + exported = _exported(QKVSplit().eval(), (torch.randn(1, 32, 4, 4),)) + permutes = [n for n in exported.main_graph.nodes if n.opcode == OpCode.PERMUTE] + assert permutes, "expected a permute node" + for node in permutes: + in_t = exported.main_graph.get_tensor(node.inputs[0]) + assert len(in_t.shape.dims) <= 4, (node.name, in_t.shape.dims) + assert len(node.attributes["dims"]) <= 4, (node.name, node.attributes) + sq = [n for n in exported.main_graph.nodes if n.name.endswith("_sq5")] + assert sq, "expected the squeeze helper node" + + +def test_5d_permute_lowes_to_ggml_permute(): + """The decomposed permute lowers to PERMUTE with concrete 4D axes.""" + torch.manual_seed(0) + exported = _exported(QKVSplit().eval(), (torch.randn(1, 32, 4, 4),)) + ggml_graph = lower_to_ggml(exported.main_graph, enable_fusion=False) + perms = [op for op in ggml_graph.nodes if op.opcode == GGMLOpCode.GGML_OP_PERMUTE] + assert perms, [op.opcode for op in ggml_graph.nodes] + for op in perms: + axes = [op.attributes[f"axis{i}"] for i in range(4)] + assert all(0 <= a <= 3 for a in axes), (op.name, axes) + assert sorted(axes) == [0, 1, 2, 3], (op.name, axes) + + +def _permute5_graph(): + g = Graph(name="perm5") + x = g.add_tensor("x", Shape.from_tuple((2, 3, 4, 5, 6)), DType.F32, StorageClass.INPUT) + o = g.add_tensor("y", Shape.from_tuple((4, 2, 5, 3, 6)), DType.F32, StorageClass.OUTPUT) + g.inputs = [x.id] + g.outputs = [o.id] + g.add_op(OpCode.PERMUTE, [x.id], [o.id], attributes={"dims": [2, 0, 3, 1, 4]}, name="perm") + return g + + +def test_lowering_rejects_inexpressible_5d_permute(): + """A 5D permute with no unit dims to squeeze must fail loudly, not scramble.""" + with pytest.raises(NotImplementedError): + lower_to_ggml(_permute5_graph(), enable_fusion=False) + + +def test_mean_chain_structure(): + """Multi-dim means chain single-dim reductions over keepdim intermediates.""" + + class MeanTwo(nn.Module): + def forward(self, x): + return x.mean(dim=(1, 3), keepdim=True) + + exported = _exported(MeanTwo().eval(), (torch.randn(2, 4, 8, 8),)) + means = [n for n in exported.main_graph.nodes if n.opcode == OpCode.MEAN] + assert len(means) == 2, [n.name for n in means] + for node in means: + assert isinstance(node.attributes["dim"], int), node.attributes + ggml_graph = lower_to_ggml(exported.main_graph, enable_fusion=False) + gmeans = [op for op in ggml_graph.nodes if op.opcode == GGMLOpCode.GGML_OP_MEAN] + assert len(gmeans) == 2, [op.name for op in ggml_graph.nodes] + assert {op.attributes["ggml_dim"] for op in gmeans} == {2, 0}, [ + (op.name, op.attributes) for op in gmeans + ] + + +def test_gelu_approximate_mapping(): + """exact (default) GELU lowers to ERF(16); tanh lowers to GELU(8).""" + + class GeluBoth(nn.Module): + def forward(self, x): + a = torch.nn.functional.gelu(x) + b = torch.nn.functional.gelu(x, approximate="tanh") + return a + b + + exported = _exported(GeluBoth().eval(), (torch.randn(2, 8),)) + gelus = {n.name: n for n in exported.main_graph.nodes if n.opcode == OpCode.GELU} + assert len(gelus) == 2, [n.opcode for n in exported.main_graph.nodes] + approx = sorted(n.attributes["approximate"] for n in gelus.values()) + assert approx == ["none", "tanh"], approx + ggml_graph = lower_to_ggml(exported.main_graph, enable_fusion=False) + codes = sorted( + op.attributes["unary_op"] + for op in ggml_graph.nodes + if op.opcode == GGMLOpCode.GGML_OP_UNARY + ) + assert codes == [8, 16], codes + + +def test_expand_as_mapping(): + """aten.expand_as maps to EXPAND with the other tensor's shape.""" + + class ExpandAs(nn.Module): + def forward(self, x, y): + return x.expand_as(y) + y + + exported = _exported(ExpandAs().eval(), (torch.randn(1, 4, 1, 1), torch.randn(1, 4, 8, 8))) + expands = [n for n in exported.main_graph.nodes if n.opcode == OpCode.EXPAND] + assert len(expands) == 1, [n.opcode for n in exported.main_graph.nodes] + out_t = exported.main_graph.get_tensor(expands[0].outputs[0]) + assert [d.evaluate({}) for d in out_t.shape.dims] == [1, 4, 8, 8] + + +def test_lowering_rejects_multi_axis_sum(): + """A multi-axis SUM must be chained by the frontend, never silently sliced.""" + from ggmlc.ir.dtype import DType + from ggmlc.ir.graph import Graph + from ggmlc.ir.op import OpCode + from ggmlc.ir.shape import Shape + from ggmlc.ir.tensor import StorageClass + + g = Graph(name="multisum") + x = g.add_tensor("x", Shape.from_tuple((1, 4, 8, 8)), DType.F32, StorageClass.INPUT) + o = g.add_tensor("y", Shape.from_tuple((1, 1, 1, 8)), DType.F32, StorageClass.OUTPUT) + g.inputs = [x.id] + g.outputs = [o.id] + g.add_op(OpCode.SUM, [x.id], [o.id], attributes={"axes": (1, 2)}, name="bad_sum") + with pytest.raises(NotImplementedError): + lower_to_ggml(g, enable_fusion=False) + + +def test_torch_multi_dim_sum_chains(): + """Multi-dim torch sums chain single-axis SUMs instead of dropping axes.""" + + class SumTwo(nn.Module): + def forward(self, x): + return x.sum(dim=(1, 2), keepdim=True) + + exported = _exported(SumTwo().eval(), (torch.randn(2, 4, 8, 8),)) + sums = [n for n in exported.main_graph.nodes if n.opcode == OpCode.SUM] + assert len(sums) == 2, [n.name for n in sums] + ggml_graph = lower_to_ggml(exported.main_graph, enable_fusion=False) + gsums = [op for op in ggml_graph.nodes if op.opcode == GGMLOpCode.GGML_OP_SUM_ROWS] + assert len(gsums) == 2, [op.name for op in ggml_graph.nodes] diff --git a/tests/ops/test_folded_slice.py b/tests/ops/test_folded_slice.py new file mode 100644 index 0000000..70c0333 --- /dev/null +++ b/tests/ops/test_folded_slice.py @@ -0,0 +1,70 @@ +"""5D stacked slices (e.g. QKV getitem) must lower to valid 4D views. + +Outer dims fold into ggml ne[3], so the slice lands on ggml dim 3 with an +offset multiplier - never an out-of-range stride index like nb[4]. +""" + +from pathlib import Path + +from ggmlc.codegen import generate_cpp_project +from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.dialect.ggml.ops import GGMLOpCode +from ggmlc.ir.dtype import DType +from ggmlc.ir.graph import Graph +from ggmlc.ir.op import OpCode +from ggmlc.ir.shape import Shape +from ggmlc.ir.tensor import StorageClass + + +def _stack_graph(batch): + g = Graph(name="qkv_split") + stacked = g.add_tensor( + "stacked", Shape.from_tuple((3, batch, 2, 4, 8)), DType.F32, StorageClass.INPUT + ) + outs = [] + for i in range(3): + o = g.add_tensor( + f"part{i}", Shape.from_tuple((batch, 2, 4, 8)), DType.F32, StorageClass.OUTPUT + ) + outs.append(o.id) + g.inputs = [stacked.id] + g.outputs = outs + for i, oid in enumerate(outs): + g.add_op( + OpCode.SLICE, + [stacked.id], + [oid], + attributes={"dim": 0, "start": i, "end": i + 1, "step": 1}, + name=f"getitem_{i}", + ) + return g + + +def _lowered_views(batch): + ggml_graph = lower_to_ggml(_stack_graph(batch), enable_fusion=False) + views = [op for op in ggml_graph.nodes if op.opcode == GGMLOpCode.GGML_OP_VIEW] + assert len(views) == 3, [op.opcode for op in ggml_graph.nodes] + return ggml_graph, views + + +def test_folded_slice_stays_within_4d(): + _, views = _lowered_views(batch=1) + for op in views: + assert op.attributes["ggml_dim"] <= 3, op.attributes + assert op.attributes.get("offset_mult", 1) == 1, op.attributes + + +def test_folded_slice_offset_multiplier(): + _, views = _lowered_views(batch=2) + for op in views: + assert op.attributes["ggml_dim"] == 3, op.attributes + assert op.attributes["offset_mult"] == 2, op.attributes + + +def test_folded_slice_codegen_has_no_oob_stride(tmp_path): + ggml_graph, _ = _lowered_views(batch=1) + proj_dir = Path(tmp_path) / "proj" + generate_cpp_project(ggml_graph, proj_dir, model_name="FoldedSlice") + text = (proj_dir / "FoldedSlice.h").read_text(encoding="utf-8") + assert "nb[4]" not in text + assert "->nb[3]" in text diff --git a/tests/ops/test_grouped_conv.py b/tests/ops/test_grouped_conv.py new file mode 100644 index 0000000..b924acc --- /dev/null +++ b/tests/ops/test_grouped_conv.py @@ -0,0 +1,51 @@ +import torch +from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.dialect.ggml.ops import GGMLOpCode +from ggmlc.frontend.pytorch import export_torch_model +from torch import nn + + +class GroupedMultiplierConv(nn.Module): + """Grouped conv with channel multiplier: groups=80, C_in=80, C_out=160. + + Not depthwise, so ggml has no single op for it: the importer must + decompose it into per-group convs instead of emitting CONV_2D_DW. + """ + + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(80, 160, 7, padding=3, groups=80, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.conv(x) + + +class TrueDepthwiseConv(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(16, 16, 3, padding=1, groups=16, bias=False) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.conv(x) + + +def test_grouped_multiplier_decomposes_without_dw(): + model = GroupedMultiplierConv().eval() + x = torch.randn(1, 80, 16, 16) + exported = export_torch_model(model, (x,), model_name="tiny_grouped", enable_fusion=False) + ggml = lower_to_ggml(exported.main_graph, enable_fusion=False) + dw = [op for op in ggml.nodes if op.opcode == GGMLOpCode.GGML_OP_CONV_2D_DW] + assert not dw, [op.name for op in dw] + convs = [op for op in ggml.nodes if op.opcode == GGMLOpCode.GGML_OP_CONV_2D] + assert len(convs) == 80, [op.name for op in convs] + + +def test_true_depthwise_stays_single_dw(): + model = TrueDepthwiseConv().eval() + x = torch.randn(1, 16, 16, 16) + exported = export_torch_model(model, (x,), model_name="tiny_dw", enable_fusion=False) + ggml = lower_to_ggml(exported.main_graph, enable_fusion=False) + dw = [op for op in ggml.nodes if op.opcode == GGMLOpCode.GGML_OP_CONV_2D_DW] + assert len(dw) == 1, [op.opcode for op in ggml.nodes] + convs = [op for op in ggml.nodes if op.opcode == GGMLOpCode.GGML_OP_CONV_2D] + assert not convs diff --git a/tests/runtime/test_interpreter_parity.py b/tests/runtime/test_interpreter_parity.py new file mode 100644 index 0000000..8f31c01 --- /dev/null +++ b/tests/runtime/test_interpreter_parity.py @@ -0,0 +1,196 @@ +"""Interpreter (nanobind runtime) vs torch parity for ops touched on this branch. + +The standalone suite covers the generated C++ path; these cover the +interpreter path over the same lowered graphs, so the two runners must +agree with each other (both are diffed against torch here). +""" + +import ggmlc +import numpy as np +import pytest +import torch +from torch import nn + +torch.manual_seed(0) + + +def _run_interpreter(model, args, atol=1e-5, rtol=1e-4): + model = model.eval() + with torch.inference_mode(): + ref = model(*args).detach().cpu().numpy() + gguf_bytes = ggmlc.compile(model=model, sample_inputs=args, model_name="parity") + runner = ggmlc.load(gguf_bytes, n_threads=2) + out = runner(*[np.ascontiguousarray(a.detach().cpu().numpy()) for a in args]) + out = np.asarray(out, dtype=np.float32) + # The runner pads outputs to 4D; compare values with ref reshaped alike. + assert out.size == ref.size, (out.shape, ref.shape) + ref = ref.reshape(out.shape) + diff = np.abs(out - ref) + tol = atol + rtol * np.abs(ref) + bad = diff > tol + assert not bad.any(), f"maxdiff={diff.max()} ref0={ref.flat[0]} out0={out.flat[0]}" + return out + + +@pytest.mark.parametrize("dim", [0, 1, 2, 3]) +def test_interpreter_mean_each_dim(dim): + """MEAN over every torch dim of a 4D input (ggml dims 3/2 use the swap path).""" + + class MeanDim(nn.Module): + def __init__(self, d): + super().__init__() + self.d = d + + def forward(self, x): + return x.mean(dim=self.d, keepdim=True) + + _run_interpreter(MeanDim(dim), (torch.randn(1, 4, 8, 8),)) + + +def test_interpreter_mean_no_keepdim(): + class MeanNoKD(nn.Module): + def forward(self, x): + return x.mean(dim=1) + + _run_interpreter(MeanNoKD(), (torch.randn(2, 4, 8, 8),)) + + +def test_interpreter_mean_two_dims(): + class MeanTwo(nn.Module): + def forward(self, x): + return x.mean(dim=(2, 3), keepdim=True) + + _run_interpreter(MeanTwo(), (torch.randn(1, 2, 4, 8),)) + + +@pytest.mark.parametrize("dim", [0, 1, 2, 3]) +def test_interpreter_sum_each_dim(dim): + class SumDim(nn.Module): + def __init__(self, d): + super().__init__() + self.d = d + + def forward(self, x): + return x.sum(dim=self.d, keepdim=True) + + _run_interpreter(SumDim(dim), (torch.randn(1, 4, 8, 8),), atol=1e-4) + + +def test_interpreter_gelu_exact(): + class GeluExact(nn.Module): + def forward(self, x): + return torch.nn.functional.gelu(x * 2.0) + + _run_interpreter(GeluExact(), (torch.randn(2, 16),)) + + +def test_interpreter_gelu_tanh(): + class GeluTanh(nn.Module): + def forward(self, x): + return torch.nn.functional.gelu(x * 2.0, approximate="tanh") + + # ggml's tanh GELU is a fast approximation of torch's tanh formula. + _run_interpreter(GeluTanh(), (torch.randn(2, 16),), atol=5e-3, rtol=1e-3) + + +def test_interpreter_qkv_split_permute(): + """QKV reshape/5D-permute/unbind plumbing must preserve head/slot order.""" + + class QKVSplit(nn.Module): + def __init__(self): + super().__init__() + self.proj = nn.Linear(32, 3 * 4 * 8, bias=False) + + def forward(self, x): + b, _c, h, w = x.shape + qkv = self.proj(x.flatten(2).transpose(1, 2)) + qkv = qkv.reshape(b, h * w, 3, 4, 8).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + return q + 2 * k + 3 * v + + _run_interpreter(QKVSplit(), (torch.randn(1, 32, 4, 4),)) + + +def test_interpreter_clamp_min_open(): + class ClampMin(nn.Module): + def forward(self, x): + return (x * 10.0).clamp_min(0.5) + + _run_interpreter(ClampMin(), (torch.randn(4, 16),)) + + +def test_interpreter_sdpa_default_scale(): + class TinyAttention(nn.Module): + def forward(self, q, k, v): + return torch.nn.functional.scaled_dot_product_attention(q, k, v) + + args = tuple(torch.randn(1, 2, 8, 16) for _ in range(3)) + _run_interpreter(TinyAttention(), args, atol=1e-4) + + +def test_interpreter_conv_bias(): + class ConvBias(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 8, 3, padding=1) + + def forward(self, x): + return self.conv(x) + + _run_interpreter(ConvBias(), (torch.randn(1, 3, 16, 16),)) + + +def test_interpreter_fused_conv_relu(): + class ConvRelu(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(3, 8, 3, padding=1) + + def forward(self, x): + return torch.relu(self.conv(x)) + + _run_interpreter(ConvRelu(), (torch.randn(1, 3, 16, 16),)) + + +def test_interpreter_expand_as(): + class ExpandAs(nn.Module): + def forward(self, x, y): + return x.expand_as(y) + y + + _run_interpreter(ExpandAs(), (torch.randn(1, 4, 1, 1), torch.randn(1, 4, 8, 8))) + + +def test_interpreter_sum_two_dims(): + class SumTwo(nn.Module): + def forward(self, x): + return x.sum(dim=(1, 2), keepdim=True) + + _run_interpreter(SumTwo(), (torch.randn(1, 4, 8, 8),), atol=1e-4) + + +def test_interpreter_sum_two_dims_no_keepdim(): + class SumTwoNoKD(nn.Module): + def forward(self, x): + return x.sum(dim=(2, 3)) + + _run_interpreter(SumTwoNoKD(), (torch.randn(1, 4, 8, 8),), atol=1e-4) + + +def test_interpreter_mean_two_dims_no_keepdim(): + class MeanTwoNoKD(nn.Module): + def forward(self, x): + return x.mean(dim=(1, 2)) + + _run_interpreter(MeanTwoNoKD(), (torch.randn(2, 4, 8, 8),)) + + +def test_interpreter_depthwise_conv(): + class Depthwise(nn.Module): + def __init__(self): + super().__init__() + self.conv = nn.Conv2d(8, 8, 3, padding=1, groups=8) + + def forward(self, x): + return self.conv(x) + + _run_interpreter(Depthwise(), (torch.randn(1, 8, 16, 16),)) diff --git a/tests/serialization/test_gguf_serialization.py b/tests/serialization/test_gguf_serialization.py index ae424f7..91c0336 100644 --- a/tests/serialization/test_gguf_serialization.py +++ b/tests/serialization/test_gguf_serialization.py @@ -40,3 +40,32 @@ def test_gguf_serialization_roundtrip_wsl(): expected = (x_val @ w.data.T + b.data).reshape(out_val.shape) np.testing.assert_allclose(out_val, expected, rtol=1e-5, atol=1e-5) + + +def test_gguf_long_tensor_names_load_natively(): + """Parameter names >= GGML_MAX_NAME (64) must be shortened consistently. + + The native loader binds weight data by name, so the tensor-info table + and the graph-spec JSON must share the shortened names. + """ + from ggmlc import _runtime + from ggmlc.serialization.gguf import deserialize_ggml_graph + + g = Graph("long_names") + x = g.add_tensor("x", Shape([1, 4]), DType.F32, StorageClass.INPUT) + long_name = "visual.trunk.stages.3.blocks.11.mlp.fc1.weight.with.extra.suffix" + assert len(long_name) >= 64 + w = g.add_tensor(long_name, Shape([4, 4]), DType.F32, StorageClass.PARAMETER) + out = g.add_tensor("out", Shape([1, 4]), DType.F32, StorageClass.ACTIVATION) + w.data = np.eye(4, dtype=np.float32) + g.add_node(OpCode.LINEAR, inputs=[x.id, w.id], outputs=[out.id]) + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = [w.id] + + ggml_graph = lower_to_ggml(g) + gguf_bytes = bytes(serialize_to_gguf(ggml_graph)) + assert gguf_bytes[:4] == b"GGUF" + _runtime.ModelLoader.load_from_bytes(gguf_bytes) # native parser accepts + g2 = deserialize_ggml_graph(gguf_bytes) # python reader agrees + assert len(g2.tensors) == len(ggml_graph.tensors) diff --git a/tests/unit/test_frontend_jax.py b/tests/unit/test_frontend_jax.py index aff48f2..04f3725 100644 --- a/tests/unit/test_frontend_jax.py +++ b/tests/unit/test_frontend_jax.py @@ -1,7 +1,10 @@ +import ggmlc import jax import jax.numpy as jnp import numpy as np +from ggmlc.dialect.ggml.lowering import lower_to_ggml from ggmlc.frontend.jax import export_jax_fn +from ggmlc.ir.op import OpCode from ggmlc.ir.tensor import StorageClass @@ -45,3 +48,49 @@ def mlp(x, w, b): assert len(params) == 2 assert g.get_tensor(g.inputs[0]).name == "x" g.validate_invariants() + + +def test_jax_multi_axis_sum_chains(): + """Multi-axis reduce_sum chains single-axis SUMs (ggml reduces one axis).""" + + def pool(x): + return jnp.sum(x, axis=(1, 2)) + + x = np.ones((2, 8, 8, 4), dtype=np.float32) + model = export_jax_fn(pool, (x,), input_names=["x"], model_name="pool") + g = model.main_graph + sums = [n for n in g.nodes if n.opcode == OpCode.SUM] + assert len(sums) == 2, [n.name for n in sums] + for node in sums: + assert node.attributes["dim"] in (1, 2), node.attributes + ggml_graph = lower_to_ggml(g, enable_fusion=False) + assert len(ggml_graph.nodes) >= len(sums) + g.validate_invariants() + + +def test_jax_asymmetric_conv_padding(): + """TF 'SAME' convs with odd total padding keep exact shapes and values.""" + + rng = np.random.default_rng(0) + w = rng.standard_normal((5, 5, 3, 8)).astype(np.float32) + dn = ("NHWC", "HWIO", "NHWC") + + def conv(x): + return jax.lax.conv_general_dilated( + x, w, (2, 2), "SAME", feature_group_count=1, dimension_numbers=dn + ) + + x = rng.standard_normal((1, 56, 56, 3)).astype(np.float32) + model = export_jax_fn(conv, (x,), input_names=["x"], model_name="aconv") + g = model.main_graph + convs = [n for n in g.nodes if n.opcode == OpCode.CONV2D] + assert convs, "expected a conv node" + pads = [n for n in g.nodes if n.opcode == OpCode.CONCAT and "pad" in n.name] + assert pads, "expected explicit zero-pad concats for the asymmetric sides" + + gguf_bytes = ggmlc.compile(model=conv, sample_inputs=(x,), model_name="aconv") + runner = ggmlc.load(gguf_bytes, n_threads=2) + out = np.asarray(runner(x)) + ref = np.asarray(conv(jax.numpy.asarray(x))) + assert out.reshape(ref.shape).shape == ref.shape + np.testing.assert_allclose(out.reshape(ref.shape), ref, atol=1e-5, rtol=1e-4)