diff --git a/pyproject.toml b/pyproject.toml index b191e7e..29ae018 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build" [project] name = "ggmlc" -version = "0.9.6" +version = "0.9.7" description = "A multi-framework neural network compiler lowering PyTorch, JAX, Flax, and Keras models to portable, high-performance GGML execution" authors = [ { name = "M. Yusuf Sarıgöz", email = "yusufsarigoz@gmail.com" }, diff --git a/python/ggmlc/__init__.py b/python/ggmlc/__init__.py index f2ef697..172150b 100644 --- a/python/ggmlc/__init__.py +++ b/python/ggmlc/__init__.py @@ -39,7 +39,9 @@ __version__ = "0.1.0" __all__ = [ + "CompileOptions", "DType", + "FusionOptions", "Graph", "ModelRunner", "OpCode", @@ -48,6 +50,7 @@ "codegen", "compile", "compile_to_bytes", + "compile_to_graph", "generate_cpp_project", "get_available_devices", "graph_to_mermaid", @@ -58,10 +61,14 @@ def __getattr__(name: str) -> Any: """Lazy-loads compiler, IR, codegen, and visualization modules on demand.""" - if name in ("compile", "compile_to_bytes", "codegen"): + if name in ("compile", "compile_to_bytes", "compile_to_graph", "codegen", "CompileOptions"): import ggmlc.compiler as compiler_mod return getattr(compiler_mod, name) + if name == "FusionOptions": + from ggmlc.transforms.fusion import FusionOptions + + return FusionOptions if name == "generate_cpp_project": import ggmlc.codegen as codegen_mod diff --git a/python/ggmlc/codegen/cpp.py b/python/ggmlc/codegen/cpp.py index 9b64f3c..35be75d 100644 --- a/python/ggmlc/codegen/cpp.py +++ b/python/ggmlc/codegen/cpp.py @@ -46,10 +46,29 @@ def __init__(self, graph: GGMLExecutionGraph, model_name: str = "model"): def generate_header(self) -> str: """Generates model.h containing weights struct and build_graph function.""" + in_descs = [ + f'"{self.graph.tensors[i].name}" [{"x".join(_dim_to_cpp_expr(d) for d in self.graph.tensors[i].ne)}]' + for i in self.graph.inputs + if i in self.graph.tensors + ] + out_descs = [ + f'"{self.graph.tensors[i].name}" [{"x".join(_dim_to_cpp_expr(d) for d in self.graph.tensors[i].ne)}]' + for i in self.graph.outputs + if i in self.graph.tensors + ] + in_summary = ", ".join(in_descs) if in_descs else "none" + out_summary = ", ".join(out_descs) if out_descs else "none" + lines: list[str] = [ "// ============================================================================", - f"// Generated C++ GGML Model Definition: {self.model_name}", - "// Automatically generated by ggmlc. Supports CPU, CUDA, and Apple Metal.", + f"// Model Architecture & Execution Graph: {self.model_name}", + "// Automatically transpiled by ggmlc. Native execution across CPU, CUDA, Metal.", + "//", + "// Topology Summary:", + f"// - Declared Inputs ({len(self.graph.inputs)}): {in_summary}", + f"// - Model Parameters ({len(self.graph.parameters)} tensors)", + f"// - Declared Outputs ({len(self.graph.outputs)}): {out_summary}", + f"// - Computational Operations: {len(self.graph.nodes)} execution graph nodes", "// ============================================================================", "#pragma once", "", @@ -178,6 +197,36 @@ def generate_header(self) -> str: "}", "", "// ----------------------------------------------------------------------------", + "// Reshape 4D contiguity helper.", + "// ----------------------------------------------------------------------------", + "inline struct ggml_tensor* reshape4d_contig(", + " struct ggml_context* ctx,", + " struct ggml_tensor* t,", + " int64_t ne0, int64_t ne1, int64_t ne2, int64_t ne3) {", + " if (!t) return nullptr;", + " if (!ggml_is_contiguous(t)) t = ggml_cont(ctx, t);", + " return ggml_reshape_4d(ctx, t, ne0, ne1, ne2, ne3);", + "}", + "", + "// ----------------------------------------------------------------------------", + "// Convolution bias matching helper.", + "// ----------------------------------------------------------------------------", + "inline struct ggml_tensor* match_conv_bias(", + " struct ggml_context* ctx,", + " struct ggml_tensor* result,", + " struct ggml_tensor* bias) {", + " if (!bias || !result) return result;", + " if (!ggml_is_contiguous(bias)) bias = ggml_cont(ctx, bias);", + " if (bias->ne[0] == result->ne[2] && bias->ne[1] == 1 && bias->ne[2] == 1) {", + " bias = reshape4d_contig(ctx, bias, 1, 1, result->ne[2], 1);", + " }", + " if (!ggml_are_same_shape(bias, result) && ggml_can_repeat(bias, result)) {", + " bias = ggml_repeat(ctx, bias, result);", + " }", + " return ggml_add(ctx, result, bias);", + "}", + "", + "// ----------------------------------------------------------------------------", "// Model Parameter Weights Structure", "// ----------------------------------------------------------------------------", "struct Weights {", @@ -256,7 +305,8 @@ def generate_header(self) -> str: "};", "", "// ----------------------------------------------------------------------------", - "// Computation Graph Builder", + "// Computation Graph Builder (build_graph)", + "// Constructs the forward pass computational Directed Acyclic Graph (DAG).", "// ----------------------------------------------------------------------------", "inline struct ggml_cgraph* build_graph(", " struct ggml_context* ctx,", @@ -272,7 +322,9 @@ def generate_header(self) -> str: " // Mapping from tensor ID to allocated computation node", " std::unordered_map tensors;", "", - " // Bind parameter tensors from weights", + " // ========================================================================", + " // Phase 1: Bind Persistent Model Weights from Weights Struct", + " // ========================================================================", ] ) @@ -282,7 +334,13 @@ def generate_header(self) -> str: lines.append(f" tensors[{t.id}] = weights.{ident};") lines.append("") - lines.append(" // Bind model inputs") + lines.append( + " // ========================================================================" + ) + lines.append(" // Phase 2: Ingest and Validate Named Model Inputs") + lines.append( + " // ========================================================================" + ) for in_id in self.graph.inputs: t = self.graph.tensors[in_id] lines.append(f' auto in_it_{in_id} = inputs.find("{t.name}");') @@ -292,11 +350,11 @@ def generate_header(self) -> str: lines.append("") lines.append( - " // ------------------------------------------------------------------------" + " // ========================================================================" ) - lines.append(" // Execution Graph Forward Pass Nodes") + lines.append(" // Phase 3: Forward Execution Graph Nodes (Operations & Activations)") lines.append( - " // ------------------------------------------------------------------------" + " // ========================================================================" ) # Emit computational nodes @@ -305,7 +363,13 @@ def generate_header(self) -> str: lines.extend(node_code) lines.append("") - lines.append(" // Build graph outputs") + lines.append( + " // ========================================================================" + ) + lines.append(" // Phase 4: Register Model Outputs with Graph Forward Expander") + lines.append( + " // ========================================================================" + ) for out_id in self.graph.outputs: lines.append(f" if (tensors.find({out_id}) != tensors.end()) {{") lines.append(f" ggml_build_forward_expand(gf, tensors[{out_id}]);") @@ -324,10 +388,274 @@ def generate_header(self) -> str: return "\n".join(lines) - def _generate_node_code(self, node: GGMLOpDef) -> list[str]: + def _get_node_semantics_and_note(self, node: GGMLOpDef) -> tuple[str, str]: + """Returns (high_level_semantics, lowering_notes) for a GGML op.""" + opc = node.opcode + attrs = node.attributes + + if opc == GGMLOpCode.GGML_OP_MUL_MAT: + if len(node.inputs) > 2: + return ( + "Linear Projection / GEMM + Bias: y = x @ W^T + b", + "GGML mul_mat consumes transposed weight as 1st argument (W, x); bias added after GEMM", + ) + return ( + "Matrix Multiplication (GEMM): y = x @ W^T", + "GGML mul_mat consumes transposed weight as 1st argument (W, x)", + ) + if opc == GGMLOpCode.GGML_OP_ADD: + return ( + "Elementwise Addition: y = a + b", + "match_broadcast handles PyTorch-to-GGML dimension broadcasting", + ) + if opc == GGMLOpCode.GGML_OP_SUB: + return ( + "Elementwise Subtraction: y = a - b", + "match_broadcast handles PyTorch-to-GGML dimension broadcasting", + ) + if opc == GGMLOpCode.GGML_OP_MUL: + return ( + "Elementwise Multiplication: y = a * b", + "match_broadcast handles PyTorch-to-GGML dimension broadcasting", + ) + if opc == GGMLOpCode.GGML_OP_DIV: + return ( + "Elementwise Division: y = a / b", + "match_broadcast handles PyTorch-to-GGML dimension broadcasting", + ) + if opc == GGMLOpCode.GGML_OP_CUSTOM_SWIGLU: + return ( + "Fused SwiGLU Activation: y = silu(gate) * up", + "Fused kernel computes SwiGLU non-linearity in a single high-efficiency memory pass", + ) + if opc == GGMLOpCode.GGML_OP_CUSTOM_RMS_NORM: + eps = attrs.get("eps", 1e-5) + return ( + f"Fused RMS Normalization: y = (x / rms(x, eps={eps})) * weight", + "Standard LLM pre/post-attention root-mean-square normalization", + ) + if opc == GGMLOpCode.GGML_OP_CUSTOM_LAYER_NORM: + eps = attrs.get("eps", 1e-5) + return ( + f"Fused Layer Normalization: y = ((x - mean) / sqrt(var + {eps})) * gamma + beta", + "Standard LayerNorm with affine transformation", + ) + if opc == GGMLOpCode.GGML_OP_CUSTOM_BIAS_GELU: + return ( + "Fused Bias + GELU Activation: y = gelu(x + bias)", + "Fused kernel computes bias addition and GELU non-linearity in single pass", + ) + if opc == GGMLOpCode.GGML_OP_ROPE: + n_dims = attrs.get("n_dims", "head_dim") + mode = attrs.get("mode", 0) + return ( + f"Rotary Position Embedding (RoPE): n_dims={n_dims}, mode={mode}", + "Applies frequency rotations to query/key projections according to token position", + ) + if opc == GGMLOpCode.GGML_OP_FLASH_ATTN_EXT: + return ( + "Scaled Dot-Product Attention (FlashAttention): softmax(Q @ K^T / sqrt(d) + mask) @ V", + "Fused multi-head attention kernel with causal masking and online softmax", + ) + if opc == GGMLOpCode.GGML_OP_CONV_2D: + s0, s1 = attrs.get("s0", 1), attrs.get("s1", 1) + p0, p1 = attrs.get("p0", 0), attrs.get("p1", 0) + return ( + f"2D Convolution: y = conv2d(x, W, stride=[{s0}, {s1}], pad=[{p0}, {p1}]) + bias", + "PyTorch NCHW tensor converted to GGML WHCN memory layout for 2D convolution", + ) + if opc == GGMLOpCode.GGML_OP_CONV_2D_DW: + s0, s1 = attrs.get("s0", 1), attrs.get("s1", 1) + p0, p1 = attrs.get("p0", 0), attrs.get("p1", 0) + return ( + f"Depthwise 2D Convolution: stride=[{s0}, {s1}], pad=[{p0}, {p1}]", + "Dedicated direct depthwise 2D convolution kernel", + ) + if opc == GGMLOpCode.GGML_OP_POOL_2D: + return ("2D Pooling (Max or Average)", "Spatial downsampling over HxW grid") + if opc == GGMLOpCode.GGML_OP_GET_ROWS: + return ( + "Embedding Lookup: y = embedding_table[token_indices]", + "Token indices flattened to 1D vector before ggml_get_rows; result reshaped back to 4D", + ) + if opc == GGMLOpCode.GGML_OP_UNARY: + u_type = attrs.get("unary_op", "gelu") + u_str = str(u_type).lower() + if "erf" in u_str or u_type == 16: + return ( + "Exact GELU Activation: y = 0.5 * x * (1 + erf(x / sqrt(2)))", + "Mapped from PyTorch torch.nn.functional.gelu(approximate='none')", + ) + if "gelu" in u_str or u_type == 8: + return ( + "Tanh-Approximated GELU Activation: y = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3)))", + "Mapped from PyTorch torch.nn.functional.gelu(approximate='tanh')", + ) + if "silu" in u_str or u_type == 10: + return ("SiLU / Swish Activation: y = x * sigmoid(x)", "GGML unary SiLU kernel") + if "relu" in u_str or u_type == 6: + return ("ReLU Activation: y = max(0, x)", "GGML unary ReLU kernel") + if "sigmoid" in u_str or u_type == 7: + return ("Sigmoid Activation: y = 1 / (1 + exp(-x))", "GGML unary Sigmoid kernel") + if "hardswish" in u_str or u_type == 11: + return ( + "Hardswish Activation: y = x * relu6(x + 3) / 6", + "GGML unary Hardswish kernel", + ) + if "hardsigmoid" in u_str or u_type == 12: + return ( + "Hardsigmoid Activation: y = relu6(x + 3) / 6", + "GGML unary Hardsigmoid kernel", + ) + return (f"Unary Activation: {u_type}", "GGML unary kernel") + if opc == GGMLOpCode.GGML_OP_REPEAT: + return ( + "Broadcast / Repeat: expand tensor to match target dimensions", + "Uses ggml_repeat_4d or reshape_4d", + ) + if opc == GGMLOpCode.GGML_OP_VIEW: + g_dim = attrs.get("ggml_dim", 0) + start = attrs.get("start", 0) + step = attrs.get("step", 1) + return ( + f"Strided Sub-Tensor View / Slice: axis={g_dim}, start={start}, step={step}", + "Zero-copy non-contiguous sub-tensor window into existing allocation", + ) + if opc == GGMLOpCode.GGML_OP_RESHAPE: + return ( + "In-Place Shape Reinterpretation", + "Adjusts tensor dimension metadata without data copies", + ) + if opc == GGMLOpCode.GGML_OP_PERMUTE: + axes = [attrs.get(f"axis{i}", i) for i in range(4)] + return ( + f"Multi-Axis Dimension Permutation: axes={axes}", + "Permutes tensor strides without copying; ggml_cont materializes contiguous layout if required", + ) + if opc == GGMLOpCode.GGML_OP_TRANSPOSE: + return ("2D Matrix Transposition: swap rows and columns", "Reorders row/column strides") + if opc == GGMLOpCode.GGML_OP_SOFT_MAX: + return ( + "Softmax Normalization: y = exp(x - max(x)) / sum(exp(x - max(x)))", + "Computes row-wise softmax", + ) + if opc == GGMLOpCode.GGML_OP_CLAMP: + min_v = attrs.get("min", "-inf") + max_v = attrs.get("max", "+inf") + return ( + f"Value Clamping: y = clamp(x, min={min_v}, max={max_v})", + "Restricts tensor elements within range", + ) + if opc == GGMLOpCode.GGML_OP_CONCAT: + dim = attrs.get("dim", 0) + return ( + f"Tensor Concatenation: concat along axis {dim}", + "Assembles multiple input tensors into one", + ) + if opc == GGMLOpCode.GGML_OP_NORM: + return ( + "L2 Normalization / Euclidean Norm along rows", + "Computes Euclidean norm of input vectors", + ) + if opc == GGMLOpCode.GGML_OP_MEAN: + g_dim = attrs.get("ggml_dim", 0) + return ( + f"Mean Reduction along axis {g_dim}", + f"GGML only reduces rows (axis 0); axis {g_dim} permuted to axis 0 before mean", + ) + if opc in (GGMLOpCode.GGML_OP_SUM, GGMLOpCode.GGML_OP_SUM_ROWS): + g_dim = attrs.get("ggml_dim", 0) + return ( + f"Sum Reduction along axis {g_dim}", + f"GGML only reduces rows (axis 0); axis {g_dim} permuted to axis 0 before sum", + ) + if opc == GGMLOpCode.GGML_OP_ARGMAX: + return ( + "Argmax Operation: y = argmax(x, dim=0)", + "Finds index of maximum value along row dimension", + ) + if opc == GGMLOpCode.GGML_OP_CONT: + return ( + "Contiguity Enforcement", + "Allocates contiguous buffer to resolve non-contiguous memory layouts", + ) + if opc == GGMLOpCode.GGML_OP_PAD: + p0, p1 = attrs.get("p0", 0), attrs.get("p1", 0) + return (f"Zero Padding: pad=[{p0}, {p1}]", "Pads spatial dimensions with zeros") + if opc == GGMLOpCode.GGML_OP_SQR: + exp = attrs.get("exponent", 2) + return (f"Power / Square Operation: y = x^{exp}", "Computes elementwise power") + if opc == GGMLOpCode.GGML_OP_SIN: + return ("Elementwise Sine: y = sin(x)", "GGML trigonometric sine kernel") + if opc == GGMLOpCode.GGML_OP_COS: + return ("Elementwise Cosine: y = cos(x)", "GGML trigonometric cosine kernel") + if opc == GGMLOpCode.GGML_OP_LOG: + return ("Elementwise Natural Logarithm: y = ln(x)", "GGML natural logarithm kernel") + if opc == GGMLOpCode.GGML_OP_GLU: + return ("Gated Linear Unit (GLU): y = a * sigmoid(b)", "GGML GLU kernel") + + return (f"Operation: {node.opcode.name}", "") + + def _format_node_header_comment(self, node: GGMLOpDef) -> list[str]: + """Generates rich, human-readable commentary documenting PyTorch source, semantics, and lowering.""" lines = [] op_name = node.name or f"node_{node.id}" - lines.append(f" // [{node.opcode.name}] {op_name} (Node {node.id})") + lines.append( + " // ------------------------------------------------------------------------" + ) + lines.append(f" // Node {node.id}: {op_name} [{node.opcode.name}]") + + semantics, note = self._get_node_semantics_and_note(node) + if semantics: + lines.append(f" // Semantics: {semantics}") + + src_op = node.attributes.get("source_op") + mod_path = node.attributes.get("module_path") + fused_pass = node.attributes.get("fused_pass") + if src_op: + lines.append(f" // Source Op: {src_op}") + if mod_path: + lines.append(f" // PyTorch Module: {mod_path}") + if fused_pass: + lines.append(f" // Optimization Pass: {fused_pass}") + + lines.append(" // Operands:") + for idx, in_id in enumerate(node.inputs): + in_t = self.graph.tensors.get(in_id) + if in_t: + role = ( + "Parameter" + if in_id in self.graph.parameters + else ("Model Input" if in_id in self.graph.inputs else "Activation") + ) + shape_str = "x".join(_dim_to_cpp_expr(d) for d in in_t.ne) + lines.append( + f' // in[{idx}]: tensors[{in_id}] ("{in_t.name}", {role}, [{shape_str}], {in_t.ggml_type.name})' + ) + else: + lines.append(f" // in[{idx}]: tensors[{in_id}]") + + for idx, out_id in enumerate(node.outputs): + out_t = self.graph.tensors.get(out_id) + if out_t: + role = "Model Output" if out_id in self.graph.outputs else "Activation" + shape_str = "x".join(_dim_to_cpp_expr(d) for d in out_t.ne) + lines.append( + f' // out[{idx}]: tensors[{out_id}] ("{out_t.name}", {role}, [{shape_str}], {out_t.ggml_type.name})' + ) + else: + lines.append(f" // out[{idx}]: tensors[{out_id}]") + + if note: + lines.append(f" // Lowering: {note}") + + lines.append( + " // ------------------------------------------------------------------------" + ) + return lines + + def _generate_node_code(self, node: GGMLOpDef) -> list[str]: + lines = self._format_node_header_comment(node) out_id = node.outputs[0] inp_vars = [f"tensors[{i}]" for i in node.inputs] @@ -458,18 +786,29 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: p1 = node.attributes.get("pad_h", 0) d0 = node.attributes.get("dilation_w", 1) d1 = node.attributes.get("dilation_h", 1) + is_1d = int(node.attributes.get("is_1d", 0)) + lines.append(f" struct ggml_tensor* c2_w_{node.id} = {inp_vars[0]};") + lines.append(f" struct ggml_tensor* c2_x_{node.id} = {inp_vars[1]};") + if is_1d: + lines.append( + f" if (c2_w_{node.id}->ne[3] == 1) c2_w_{node.id} = reshape4d_contig(ctx, c2_w_{node.id}, c2_w_{node.id}->ne[0], 1, c2_w_{node.id}->ne[1], c2_w_{node.id}->ne[2]);" + ) + lines.append( + f" if (c2_x_{node.id}->ne[3] == 1) c2_x_{node.id} = reshape4d_contig(ctx, c2_x_{node.id}, c2_x_{node.id}->ne[0], 1, c2_x_{node.id}->ne[1], c2_x_{node.id}->ne[2]);" + ) lines.append( - f" tensors[{out_id}] = ggml_conv_2d(ctx, {inp_vars[0]}, {inp_vars[1]}, {s0}, {s1}, {p0}, {p1}, {d0}, {d1});" + f" tensors[{out_id}] = ggml_conv_2d(ctx, c2_w_{node.id}, c2_x_{node.id}, {s0}, {s1}, {p0}, {p1}, {d0}, {d1});" ) if len(inp_vars) > 2: lines.append( - f" auto bcb_{node.id} = match_broadcast(ctx, tensors[{out_id}], {inp_vars[2]});" - ) - lines.append( - f" tensors[{out_id}] = ggml_add(ctx, bcb_{node.id}.first, bcb_{node.id}.second);" + f" tensors[{out_id}] = match_conv_bias(ctx, tensors[{out_id}], {inp_vars[2]});" ) if node.attributes.get("fused_relu", 0): lines.append(f" tensors[{out_id}] = ggml_relu(ctx, tensors[{out_id}]);") + if is_1d: + lines.append( + f" if (tensors[{out_id}] && tensors[{out_id}]->ne[1] == 1) tensors[{out_id}] = reshape4d_contig(ctx, tensors[{out_id}], tensors[{out_id}]->ne[0], tensors[{out_id}]->ne[2], tensors[{out_id}]->ne[3], 1);" + ) elif node.opcode == GGMLOpCode.GGML_OP_CONV_2D_DW: s0 = node.attributes.get("stride_w", 1) s1 = node.attributes.get("stride_h", 1) @@ -486,13 +825,14 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: ) if len(inp_vars) > 2: lines.append( - f" auto bcb_{node.id} = match_broadcast(ctx, tensors[{out_id}], {inp_vars[2]});" - ) - lines.append( - f" tensors[{out_id}] = ggml_add(ctx, bcb_{node.id}.first, bcb_{node.id}.second);" + f" tensors[{out_id}] = match_conv_bias(ctx, tensors[{out_id}], {inp_vars[2]});" ) if node.attributes.get("fused_relu", 0): lines.append(f" tensors[{out_id}] = ggml_relu(ctx, tensors[{out_id}]);") + if is_1d: + lines.append( + f" if (tensors[{out_id}] && tensors[{out_id}]->ne[1] == 1) tensors[{out_id}] = reshape4d_contig(ctx, tensors[{out_id}], tensors[{out_id}]->ne[0], tensors[{out_id}]->ne[2], tensors[{out_id}]->ne[3], 1);" + ) elif node.opcode == GGMLOpCode.GGML_OP_POOL_2D: is_max = node.attributes.get("is_max", 0) != 0 pool_enum = "GGML_OP_POOL_MAX" if is_max else "GGML_OP_POOL_AVG" @@ -517,9 +857,25 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: if len(inp_vars) == 1: out_t = self.graph.tensors[out_id] ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] + lines.append(f" struct ggml_tensor* rep_in_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(rep_in_{node.id})) rep_in_{node.id} = ggml_cont(ctx, rep_in_{node.id});" + ) + lines.append( + f" int64_t rep_in_el_{node.id} = rep_in_{node.id}->ne[0] * rep_in_{node.id}->ne[1] * rep_in_{node.id}->ne[2] * rep_in_{node.id}->ne[3];" + ) + lines.append( + f" int64_t rep_out_el_{node.id} = (int64_t)({ne_strs[0]}) * (int64_t)({ne_strs[1]}) * (int64_t)({ne_strs[2]}) * (int64_t)({ne_strs[3]});" + ) + lines.append(f" if (rep_in_el_{node.id} == rep_out_el_{node.id}) {{") + lines.append( + f" tensors[{out_id}] = ggml_reshape_4d(ctx, rep_in_{node.id}, {', '.join(ne_strs)});" + ) + lines.append(" } else {") lines.append( - f" tensors[{out_id}] = ggml_repeat_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)});" + f" tensors[{out_id}] = ggml_repeat_4d(ctx, rep_in_{node.id}, {', '.join(ne_strs)});" ) + lines.append(" }") else: lines.append( f" tensors[{out_id}] = ggml_repeat(ctx, {inp_vars[0]}, {inp_vars[1]});" @@ -611,6 +967,8 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: f" tensors[{out_id}] = ggml_rope(ctx, {inp_vars[0]}, {inp_vars[1]}, {n_dims_expr}, {mode});" ) elif node.opcode == GGMLOpCode.GGML_OP_GET_ROWS: + out_t = self.graph.tensors[out_id] + ne_strs = [_dim_to_cpp_expr(d) for d in out_t.ne] lines.append(f" struct ggml_tensor* idx_{node.id} = {inp_vars[1]};") lines.append( f" if (idx_{node.id}->type != GGML_TYPE_I32) idx_{node.id} = ggml_cast(ctx, idx_{node.id}, GGML_TYPE_I32);" @@ -619,7 +977,19 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: f" if (!ggml_is_contiguous(idx_{node.id})) idx_{node.id} = ggml_cont(ctx, idx_{node.id});" ) lines.append( - f" tensors[{out_id}] = ggml_get_rows(ctx, {inp_vars[0]}, idx_{node.id});" + f" int64_t total_idx_{node.id} = idx_{node.id}->ne[0] * idx_{node.id}->ne[1] * idx_{node.id}->ne[2] * idx_{node.id}->ne[3];" + ) + lines.append( + f" struct ggml_tensor* idx_flat_{node.id} = ggml_reshape_1d(ctx, idx_{node.id}, total_idx_{node.id});" + ) + lines.append( + f" struct ggml_tensor* gr_{node.id} = ggml_get_rows(ctx, {inp_vars[0]}, idx_flat_{node.id});" + ) + lines.append( + f" if (!ggml_is_contiguous(gr_{node.id})) gr_{node.id} = ggml_cont(ctx, gr_{node.id});" + ) + lines.append( + f" tensors[{out_id}] = ggml_reshape_4d(ctx, gr_{node.id}, {', '.join(ne_strs)});" ) elif node.opcode == GGMLOpCode.GGML_OP_CONCAT: dim = node.attributes.get("ggml_dim", node.attributes.get("dim", 0)) @@ -759,13 +1129,33 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: start = node.attributes.get("start", 0) ggml_dim = node.attributes.get("ggml_dim", 0) mult = node.attributes.get("offset_mult", 1) + step = node.attributes.get("step", 1) if mult == 1: offset = f"{start} * {inp_vars[0]}->nb[{ggml_dim}]" else: offset = f"{start} * {mult} * {inp_vars[0]}->nb[{ggml_dim}]" + nb1 = ( + f"{inp_vars[0]}->nb[1] * {step}" + if ggml_dim == 1 and step != 1 + else f"{inp_vars[0]}->nb[1]" + ) + nb2 = ( + f"{inp_vars[0]}->nb[2] * {step}" + if ggml_dim == 2 and step != 1 + else f"{inp_vars[0]}->nb[2]" + ) + nb3 = ( + f"{inp_vars[0]}->nb[3] * {step}" + if ggml_dim == 3 and step != 1 + else f"{inp_vars[0]}->nb[3]" + ) lines.append( - f" tensors[{out_id}] = ggml_view_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)}, {inp_vars[0]}->nb[1], {inp_vars[0]}->nb[2], {inp_vars[0]}->nb[3], {offset});" + f" tensors[{out_id}] = ggml_view_4d(ctx, {inp_vars[0]}, {', '.join(ne_strs)}, {nb1}, {nb2}, {nb3}, {offset});" ) + if out_id in self.graph.outputs: + lines.append( + f" if (!ggml_is_contiguous(tensors[{out_id}])) tensors[{out_id}] = ggml_cont(ctx, tensors[{out_id}]);" + ) elif node.opcode == GGMLOpCode.GGML_OP_CUSTOM_BIAS_GELU: lines.append(" #if defined(GGML_USE_CUDA)") lines.append( @@ -813,15 +1203,57 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: ) lines.append(" #endif") elif node.opcode == GGMLOpCode.GGML_OP_CUSTOM_SWIGLU: - lines.append(" #if defined(GGML_USE_CUDA)") + if len(inp_vars) == 1: + swapped = node.attributes.get("swapped", 0) + fn = "ggml_swiglu_swapped" if swapped else "ggml_swiglu" + lines.append(f" tensors[{out_id}] = {fn}(ctx, {inp_vars[0]});") + else: + lines.append(" #if defined(GGML_USE_CUDA)") + lines.append( + f" tensors[{out_id}] = ggml_mul(ctx, ggml_silu(ctx, {inp_vars[0]}), {inp_vars[1]});" + ) + lines.append(" #else") + lines.append( + f" tensors[{out_id}] = ggml_map_custom2(ctx, {inp_vars[0]}, {inp_vars[1]}, ggmlc_compute_forward_swiglu, GGML_N_TASKS_MAX, nullptr);" + ) + lines.append(" #endif") + elif node.opcode == GGMLOpCode.GGML_OP_PAD: + p0 = node.attributes.get("pad_w", 0) + p1 = node.attributes.get("pad_h", 0) + p2 = node.attributes.get("pad_c", 0) + p3 = node.attributes.get("pad_n", 0) lines.append( - f" tensors[{out_id}] = ggml_mul(ctx, ggml_silu(ctx, {inp_vars[0]}), {inp_vars[1]});" + f" tensors[{out_id}] = ggml_pad(ctx, {inp_vars[0]}, {p0}, {p1}, {p2}, {p3});" ) - lines.append(" #else") + elif node.opcode == GGMLOpCode.GGML_OP_SQR: + exp = node.attributes.get("exponent", node.attributes.get("y", 2.0)) + lines.append(f" struct ggml_tensor* sqr_in_{node.id} = {inp_vars[0]};") lines.append( - f" tensors[{out_id}] = ggml_map_custom2(ctx, {inp_vars[0]}, {inp_vars[1]}, ggmlc_compute_forward_swiglu, GGML_N_TASKS_MAX, nullptr);" + f" if (!ggml_is_contiguous(sqr_in_{node.id})) sqr_in_{node.id} = ggml_cont(ctx, sqr_in_{node.id});" ) - lines.append(" #endif") + if float(exp) == 3.0: + lines.append( + f" tensors[{out_id}] = ggml_mul(ctx, sqr_in_{node.id}, ggml_sqr(ctx, sqr_in_{node.id}));" + ) + else: + lines.append(f" tensors[{out_id}] = ggml_sqr(ctx, sqr_in_{node.id});") + elif node.opcode in ( + GGMLOpCode.GGML_OP_SIN, + GGMLOpCode.GGML_OP_COS, + GGMLOpCode.GGML_OP_LOG, + ): + fn_name = { + GGMLOpCode.GGML_OP_SIN: "ggml_sin", + GGMLOpCode.GGML_OP_COS: "ggml_cos", + GGMLOpCode.GGML_OP_LOG: "ggml_log", + }[node.opcode] + lines.append(f" struct ggml_tensor* t_in_{node.id} = {inp_vars[0]};") + lines.append( + f" if (!ggml_is_contiguous(t_in_{node.id})) t_in_{node.id} = ggml_cont(ctx, t_in_{node.id});" + ) + lines.append(f" tensors[{out_id}] = {fn_name}(ctx, t_in_{node.id});") + elif node.opcode == GGMLOpCode.GGML_OP_GLU: + lines.append(f" tensors[{out_id}] = ggml_swiglu(ctx, {inp_vars[0]});") else: raise NotImplementedError( f"no C++ emission for {node.opcode.name} " @@ -833,6 +1265,29 @@ def _generate_node_code(self, node: GGMLOpDef) -> list[str]: def generate_main(self) -> str: """Generates ggmlc_main.cpp standalone execution entry point.""" + input_init_lines = [] + input_feed_lines = [] + for in_id in self.graph.inputs: + t = self.graph.tensors[in_id] + ident = _sanitize_ident(t.name) + shape_str = ", ".join(_dim_to_cpp_expr(d) for d in t.ne) + input_init_lines.append( + f" struct ggml_tensor* in_{ident} = ggml_new_tensor_4d(ctx, static_cast({int(t.ggml_type)}), {shape_str});" + ) + input_init_lines.append(f' ggml_set_name(in_{ident}, "{t.name}");') + input_init_lines.append(f' inputs["{t.name}"] = in_{ident};') + input_feed_lines.append( + f" if (in_{ident}) {{ std::vector dummy_buf_{ident}(ggml_nbytes(in_{ident}), 0); ggml_backend_tensor_set(in_{ident}, dummy_buf_{ident}.data(), 0, ggml_nbytes(in_{ident})); }}" + ) + + if not input_init_lines: + input_init_lines.append(" // No explicit model inputs declared") + if not input_feed_lines: + input_feed_lines.append(" // No input data buffers to initialize") + + inits_str = "\n".join(input_init_lines) + feeds_str = "\n".join(input_feed_lines) + return f"""// ============================================================================ // Standalone GGML Backend Model Runner: {self.model_name} // Automatically generated by ggmlc. Supports CPU, CUDA, and Apple Metal. @@ -844,9 +1299,13 @@ def generate_main(self) -> str: #include #include #include +#include int main(int argc, char** argv) {{ - std::string gguf_path = "model.gguf"; + std::string gguf_path = "{self.model_name}.gguf"; + if (std::ifstream f("model.gguf"); f.good()) {{ + gguf_path = "model.gguf"; + }} std::string device = "auto"; int n_threads = 4; @@ -919,9 +1378,7 @@ def generate_main(self) -> str: // 4. Prepare input nodes std::unordered_map inputs; - struct ggml_tensor* in_tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 16); - ggml_set_name(in_tensor, "input"); - inputs["input"] = in_tensor; +{inits_str} // 5. Build computation graph struct ggml_cgraph* gf = {self.model_name}::build_graph(ctx, weights, inputs); @@ -942,8 +1399,7 @@ def generate_main(self) -> str: }} // 8. Initialize input data via backend transfer - std::vector dummy_data(16, 1.0f); - ggml_backend_tensor_set(in_tensor, dummy_data.data(), 0, 16 * sizeof(float)); +{feeds_str} // 9. Execute computation on target backend auto t0 = std::chrono::high_resolution_clock::now(); @@ -979,18 +1435,23 @@ def generate_cmakelists(self) -> str: find_package(Threads REQUIRED) -if (ENABLE_CUDA) - enable_language(CUDA) - add_compile_definitions(GGML_USE_CUDA) - set(GGML_BACKEND_LIBS ggml ggml-base ggml-cpu ggml-cuda Threads::Threads) -elseif (ENABLE_METAL) - find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) - find_library(METAL_FRAMEWORK Metal REQUIRED) - find_library(METALKIT_FRAMEWORK MetalKit REQUIRED) - add_compile_definitions(GGML_USE_METAL) - set(GGML_BACKEND_LIBS ggml ggml-base ggml-cpu ggml-metal ${{FOUNDATION_FRAMEWORK}} ${{METAL_FRAMEWORK}} ${{METALKIT_FRAMEWORK}} Threads::Threads) +find_package(ggml CONFIG QUIET) +if (ggml_FOUND) + set(GGML_BACKEND_LIBS ggml::ggml Threads::Threads) else() - set(GGML_BACKEND_LIBS ggml_lib Threads::Threads) + if (ENABLE_CUDA) + enable_language(CUDA) + add_compile_definitions(GGML_USE_CUDA) + set(GGML_BACKEND_LIBS ggml ggml-base ggml-cpu ggml-cuda Threads::Threads) + elseif (ENABLE_METAL) + find_library(FOUNDATION_FRAMEWORK Foundation REQUIRED) + find_library(METAL_FRAMEWORK Metal REQUIRED) + find_library(METALKIT_FRAMEWORK MetalKit REQUIRED) + add_compile_definitions(GGML_USE_METAL) + set(GGML_BACKEND_LIBS ggml ggml-base ggml-cpu ggml-metal ${{FOUNDATION_FRAMEWORK}} ${{METAL_FRAMEWORK}} ${{METALKIT_FRAMEWORK}} Threads::Threads) + else() + set(GGML_BACKEND_LIBS ggml ggml-base ggml-cpu Threads::Threads) + endif() endif() add_executable({self.model_name}_run diff --git a/python/ggmlc/compiler.py b/python/ggmlc/compiler.py index c37681d..2b9a593 100644 --- a/python/ggmlc/compiler.py +++ b/python/ggmlc/compiler.py @@ -2,91 +2,60 @@ from __future__ import annotations +from dataclasses import dataclass from pathlib import Path from typing import Any from ggmlc.codegen.cpp import generate_cpp_project -from ggmlc.dialect.ggml.lowering import lower_to_ggml +from ggmlc.dialect.ggml.lowering import GGMLExecutionGraph, lower_to_ggml from ggmlc.ir.dtype import DType from ggmlc.ir.graph import Graph from ggmlc.quantization import quantize_graph_parameters from ggmlc.transforms import create_standard_optimization_pipeline +from ggmlc.transforms.fusion import FusionOptions -def compile( +@dataclass +class CompileOptions: + """Structured configuration options for compiling models in ggmlc.""" + + model_name: str = "model" + enable_optimizations: bool = True + enable_fusion: bool = True + fusion_options: FusionOptions | dict[str, bool] | None = None + quantize: str | DType | None = None + dynamic_shapes: tuple[dict[int, Any], ...] | dict[str, Any] | None = None + extra_metadata: dict[str, Any] | None = None + tasks: str | list[str] | None = None + + +def compile_to_graph( model: Any, sample_inputs: tuple[Any, ...] | list[Any] | None = None, - output: str | Path | None = None, dynamic_shapes: tuple[dict[int, Any], ...] | dict[str, Any] | None = None, model_name: str = "model", enable_optimizations: bool = True, enable_fusion: bool = True, - fusion_options: dict[str, bool] | None = None, + fusion_options: FusionOptions | dict[str, bool] | None = None, quantize: str | DType | None = None, - return_runner: bool = False, - pipeline: Any = None, - tasks: str | list[str] | None = None, - extra_metadata: dict[str, Any] | None = None, + release_module_storage: bool = False, **kwargs: Any, -) -> Path | bytes | Any: - """Compiles a PyTorch or JAX neural network model into a standard GGUF v3 binary container. - - This function ingests the source model into Canonical IR, runs standard optimization - passes (constant folding, dead-code elimination, redundant cast pruning), performs - target dialect lowering to GGML, applies optional parameter quantization (Q4_0, Q8_0), - and serializes the result into a standardized GGUF v3 file. - - Args: - model: PyTorch model (nn.Module or ExportedProgram), JAX callable, or IR Graph. - sample_inputs: Sample input tensors matching the model's forward signature. - output: Optional file path to stream the .gguf binary container directly to disk. - dynamic_shapes: Dynamic shape specifications (e.g. torch.export.Dim). - model_name: Identifier name embedded in graph and GGUF metadata. - enable_optimizations: If True, applies standard IR graph optimization passes. - enable_fusion: If True, lowers composite subgraphs into high-performance fused ops. - fusion_options: Optional granular flags for specific fusion patterns. - quantize: Optional quantization format ('f16', 'q4_0', 'q8_0', DType.F16, DType.Q4_0, DType.Q8_0). - return_runner: If True, automatically loads and returns an instantiated ModelRunner. - pipeline: Optional multimodal pipeline preprocessor (e.g. VisionPreprocessor or BPETokenizer) - whose schema metadata will be embedded into GGUF headers. - tasks: Explicit task or list of tasks supported by this model (e.g. 'classification', - ['embedding', 'similarity'], or 'text-generation') embedded into 'ggmlc.tasks'. - extra_metadata: Optional custom key-value metadata dictionary embedded into the GGUF file. - **kwargs: Additional framework-specific keyword arguments. - - Returns: - Path to the saved .gguf file (if output is provided), ModelRunner (if return_runner=True), - or raw GGUF v3 bytes (if output is None and return_runner=False). +) -> GGMLExecutionGraph: + """Ingests, optimizes, and lowers a neural network model into a GGMLExecutionGraph. - Example: - >>> import ggmlc, torch, torchvision.models as models - >>> model = models.resnet18().eval() - >>> x = torch.randn(1, 3, 224, 224) - >>> # Stream directly to disk - >>> model_path = ggmlc.compile(model, (x,), output="resnet18.gguf") - >>> runner = ggmlc.load(model_path) + This function serves as the single authoritative frontend pipeline for both + direct GGUF binary serialization (ggmlc.compile) and standalone C++ code + generation (ggmlc.codegen), guaranteeing identical ingestion, canonical IR + optimizations, and GGML lowering across both compilation targets. """ - release_module_storage = bool(kwargs.pop("release_module_storage", False)) - - # Normalize fusion_options before export so the PyTorch exporter does not - # bake in default horizontal fusion that later A/B flags cannot undo. - from ggmlc.transforms.fusion import FusionOptions as _FusionOptions - - if fusion_options is None or isinstance(fusion_options, _FusionOptions): - pass - elif isinstance(fusion_options, dict): - normalized = _FusionOptions() - for k, v in fusion_options.items(): - if hasattr(normalized, k): - setattr(normalized, k, v) - fusion_options = normalized - else: - raise TypeError(f"fusion_options must be FusionOptions or dict, got {type(fusion_options)}") + opts = FusionOptions.from_dict(fusion_options) # 1. Ingest model into Canonical IR Graph canonical_graph: Graph if isinstance(model, Graph): canonical_graph = model + elif hasattr(model, "main_graph") and isinstance(model.main_graph, Graph): + canonical_graph = model.main_graph elif hasattr(model, "graph_module") or hasattr(model, "module"): # ExportedProgram or nn.Module from ggmlc.frontend.pytorch import export_torch_model @@ -99,10 +68,14 @@ def compile( dynamic_shapes=dynamic_shapes, model_name=model_name, enable_fusion=enable_fusion, - fusion_options=fusion_options, + fusion_options=opts, release_module_storage=release_module_storage, ) canonical_graph = exported.main_graph + elif hasattr(model, "eqns"): # JAX ClosedJaxpr + from ggmlc.frontend.jax.importer import import_jaxpr + + canonical_graph = import_jaxpr(model, graph_name=model_name) elif callable(model) and not hasattr(model, "parameters"): # JAX function or callable import jax @@ -127,7 +100,7 @@ def compile( dynamic_shapes=dynamic_shapes, model_name=model_name, enable_fusion=enable_fusion, - fusion_options=fusion_options, + fusion_options=opts, release_module_storage=release_module_storage, ) canonical_graph = exported.main_graph @@ -137,7 +110,7 @@ def compile( # 2. Run Canonical IR Graph Optimizations if enable_optimizations: opt_pipeline = create_standard_optimization_pipeline( - enable_fusion=enable_fusion, options=fusion_options + enable_fusion=enable_fusion, options=opts ) opt_result = opt_pipeline.run(canonical_graph) canonical_graph = opt_result.graph @@ -146,7 +119,7 @@ def compile( ggml_graph = lower_to_ggml( canonical_graph, enable_fusion=enable_fusion, - fusion_options=fusion_options, + fusion_options=opts, ) # 4. Apply Block Quantization (Optional) @@ -159,7 +132,78 @@ def compile( gc.collect() ggml_graph, _ = quantize_graph_parameters(ggml_graph, target_dtype=quantize) - # 5. Extract metadata from pipeline and tasks if provided + return ggml_graph + + +def compile( + model: Any, + sample_inputs: tuple[Any, ...] | list[Any] | None = None, + output: str | Path | None = None, + dynamic_shapes: tuple[dict[int, Any], ...] | dict[str, Any] | None = None, + model_name: str = "model", + enable_optimizations: bool = True, + enable_fusion: bool = True, + fusion_options: FusionOptions | dict[str, bool] | None = None, + quantize: str | DType | None = None, + return_runner: bool = False, + pipeline: Any = None, + tasks: str | list[str] | None = None, + extra_metadata: dict[str, Any] | None = None, + **kwargs: Any, +) -> Path | bytes | Any: + """Compiles a PyTorch or JAX neural network model into a standard GGUF v3 binary container. + + This function ingests the source model into Canonical IR, runs standard optimization + passes (constant folding, dead-code elimination, redundant cast pruning), performs + target dialect lowering to GGML, applies optional parameter quantization (Q4_0, Q8_0), + and serializes the result into a standardized GGUF v3 file. + + Args: + model: PyTorch model (nn.Module or ExportedProgram), JAX callable, or IR Graph. + sample_inputs: Sample input tensors matching the model's forward signature. + output: Optional file path to stream the .gguf binary container directly to disk. + dynamic_shapes: Dynamic shape specifications (e.g. torch.export.Dim). + model_name: Identifier name embedded in graph and GGUF metadata. + enable_optimizations: If True, applies standard IR graph optimization passes. + enable_fusion: If True, lowers composite subgraphs into high-performance fused ops. + fusion_options: Optional granular flags for specific fusion patterns. + quantize: Optional quantization format ('f16', 'q4_0', 'q8_0', DType.F16, DType.Q4_0, DType.Q8_0). + return_runner: If True, automatically loads and returns an instantiated ModelRunner. + pipeline: Optional multimodal pipeline preprocessor (e.g. VisionPreprocessor or BPETokenizer) + whose schema metadata will be embedded into GGUF headers. + tasks: Explicit task or list of tasks supported by this model (e.g. 'classification', + ['embedding', 'similarity'], or 'text-generation') embedded into 'ggmlc.tasks'. + extra_metadata: Optional custom key-value metadata dictionary embedded into the GGUF file. + **kwargs: Additional framework-specific keyword arguments. + + Returns: + Path to the saved .gguf file (if output is provided), ModelRunner (if return_runner=True), + or raw GGUF v3 bytes (if output is None and return_runner=False). + + Example: + >>> import ggmlc, torch, torchvision.models as models + >>> model = models.resnet18().eval() + >>> x = torch.randn(1, 3, 224, 224) + >>> # Stream directly to disk + >>> model_path = ggmlc.compile(model, (x,), output="resnet18.gguf") + >>> runner = ggmlc.load(model_path) + """ + release_module_storage = bool(kwargs.pop("release_module_storage", False)) + + ggml_graph = compile_to_graph( + model=model, + sample_inputs=sample_inputs, + dynamic_shapes=dynamic_shapes, + model_name=model_name, + enable_optimizations=enable_optimizations, + enable_fusion=enable_fusion, + fusion_options=fusion_options, + quantize=quantize, + release_module_storage=release_module_storage, + **kwargs, + ) + + # Extract metadata from pipeline and tasks if provided combined_metadata: dict[str, Any] = dict(extra_metadata or {}) if pipeline is not None: if hasattr(pipeline, "to_gguf_metadata"): @@ -171,7 +215,7 @@ def compile( task_list = [tasks] if isinstance(tasks, str) else [str(t) for t in tasks] combined_metadata["ggmlc.tasks"] = task_list - # 6. Stream directly to file or serialize to memory + # Stream directly to file or serialize to memory from ggmlc.runtime.runner import load from ggmlc.serialization.gguf import save_to_gguf, serialize_to_gguf @@ -186,10 +230,9 @@ def compile( from ggmlc.frontend.pytorch.exporter import cleanup_released_storage from ggmlc.serialization.spill import cleanup_spills - for graph in (canonical_graph, ggml_graph): - for tensor in getattr(graph, "tensors", {}).values(): - tensor.data = None - del canonical_graph, ggml_graph + for tensor in getattr(ggml_graph, "tensors", {}).values(): + tensor.data = None + del ggml_graph gc.collect() cleanup_released_storage() cleanup_spills() @@ -211,7 +254,7 @@ def compile_to_bytes( model_name: str = "model", enable_optimizations: bool = True, enable_fusion: bool = True, - fusion_options: dict[str, bool] | None = None, + fusion_options: FusionOptions | dict[str, bool] | None = None, quantize: str | DType | None = None, **kwargs: Any, ) -> bytes: @@ -237,11 +280,13 @@ def codegen( model: Any, sample_inputs: tuple[Any, ...] | list[Any], output_dir: str | Path, - dynamic_shapes: tuple[dict[int, Any], ...] | None = None, + dynamic_shapes: tuple[dict[int, Any], ...] | dict[str, Any] | None = None, model_name: str = "model", enable_optimizations: bool = True, enable_fusion: bool = True, - fusion_options: dict[str, bool] | None = None, + fusion_options: FusionOptions | dict[str, bool] | None = None, + quantize: str | DType | None = None, + **kwargs: Any, ) -> Path: """Transpiles a neural network graph into a standalone, human-readable C++ project. @@ -249,7 +294,7 @@ def codegen( and CMakeLists.txt ready for native compilation. Args: - model: PyTorch model or exported program. + model: PyTorch model, JAX callable, or IR Graph. sample_inputs: Sample input tensors matching model input signature. output_dir: Destination directory for the generated C++ project. dynamic_shapes: Optional dynamic shape constraints. @@ -257,60 +302,22 @@ def codegen( enable_optimizations: If True, applies standard IR optimizations. enable_fusion: If True, lowers composite subgraphs to fused ops. fusion_options: Optional granular fusion options. + quantize: Optional quantization format ('f16', 'q4_0', 'q8_0', etc.). + **kwargs: Additional frontend export keyword arguments. Returns: Path to the generated project directory. """ - from ggmlc.dialect.ggml.lowering import FusionOptions, lower_to_ggml - from ggmlc.ir.graph import Graph - - inputs_tuple = tuple(sample_inputs) if isinstance(sample_inputs, list) else sample_inputs - - if isinstance(model, Graph): - canonical_graph = model - elif hasattr(model, "eval") or ( - isinstance(model, type) or (callable(model) and hasattr(model, "state_dict")) - ): - try: - from ggmlc.frontend.pytorch import export_torch_model - - exported = export_torch_model( - model, inputs_tuple, dynamic_shapes=dynamic_shapes, model_name=model_name - ) - canonical_graph = exported.main_graph - except (AttributeError, TypeError, ImportError, RuntimeError): - from ggmlc.frontend.jax.exporter import export_jax_fn - - exported = export_jax_fn(model, inputs_tuple, model_name=model_name) - canonical_graph = exported.main_graph - elif hasattr(model, "eqns"): # JAX ClosedJaxpr - from ggmlc.frontend.jax.importer import import_jaxpr - - canonical_graph = import_jaxpr(model, graph_name=model_name) - elif callable(model): - from ggmlc.frontend.jax.exporter import export_jax_fn - - exported = export_jax_fn(model, inputs_tuple, model_name=model_name) - canonical_graph = exported.main_graph - else: - raise TypeError(f"Unsupported model type for codegen: {type(model)}") - - f_opts = FusionOptions() - if fusion_options: - for k, v in fusion_options.items(): - if hasattr(f_opts, k): - setattr(f_opts, k, v) - - if enable_optimizations: - from ggmlc.transforms import create_standard_optimization_pipeline - - pipeline = create_standard_optimization_pipeline() - canonical_graph = pipeline(canonical_graph) - - ggml_graph = lower_to_ggml( - canonical_graph, + ggml_graph = compile_to_graph( + model=model, + sample_inputs=sample_inputs, + dynamic_shapes=dynamic_shapes, + model_name=model_name, + enable_optimizations=enable_optimizations, enable_fusion=enable_fusion, - fusion_options=f_opts, + fusion_options=fusion_options, + quantize=quantize, + **kwargs, ) generate_cpp_project( diff --git a/python/ggmlc/frontend/pytorch/importer.py b/python/ggmlc/frontend/pytorch/importer.py index dba6ca8..11bfa96 100644 --- a/python/ggmlc/frontend/pytorch/importer.py +++ b/python/ggmlc/frontend/pytorch/importer.py @@ -763,6 +763,21 @@ def _get_or_create_tensor(arg, default_name): # Collect input tensor IDs and attributes input_tensor_ids: list[int] = [] attributes: dict[str, Any] = {} + if hasattr(node.target, "__name__"): + attributes["source_op"] = f"aten.{node.target.__name__}" + else: + attributes["source_op"] = str(node.target) + + if "nn_module_stack" in node.meta: + stack = node.meta["nn_module_stack"] + if isinstance(stack, dict) and stack: + mod_key = list(stack.keys())[-1] + mod_val = list(stack.values())[-1] + cls_name = "" + if isinstance(mod_val, tuple) and len(mod_val) >= 2: + cls_obj = mod_val[1] + cls_name = cls_obj.__name__ if hasattr(cls_obj, "__name__") else str(cls_obj) + attributes["module_path"] = f"{mod_key} ({cls_name})" if cls_name else str(mod_key) if opcode == OpCode.TRANSPOSE: # aten.transpose.int(self, dim0, dim1) or aten.t(self) diff --git a/python/ggmlc/transforms/fusion.py b/python/ggmlc/transforms/fusion.py index f3b9656..5079bfd 100644 --- a/python/ggmlc/transforms/fusion.py +++ b/python/ggmlc/transforms/fusion.py @@ -5,7 +5,8 @@ import os import re import tempfile -from dataclasses import dataclass +from dataclasses import dataclass, fields +from typing import Any import numpy as np @@ -67,6 +68,40 @@ class FusionOptions: # stock CUDA fused kernels stay intact. enable_bake_affine: bool = True + @classmethod + def from_dict(cls, d: dict[str, Any] | FusionOptions | None) -> FusionOptions: + """Constructs and validates a FusionOptions instance. + + Args: + d: Dictionary of option name to boolean, FusionOptions instance, or None. + + Returns: + Validated FusionOptions instance. + + Raises: + ValueError: If unrecognized fusion option keys are provided. + TypeError: If input is not a dict or FusionOptions. + """ + if d is None: + return cls() + if isinstance(d, cls): + return d + if not isinstance(d, dict): + raise TypeError(f"Expected dict or FusionOptions, got {type(d).__name__}") + + valid_fields = {f.name for f in fields(cls)} + unknown = set(d.keys()) - valid_fields + if unknown: + raise ValueError( + f"Unknown fusion option(s): {sorted(unknown)}. " + f"Valid options are: {sorted(valid_fields)}" + ) + return cls(**{k: bool(v) for k, v in d.items()}) + + def to_dict(self) -> dict[str, bool]: + """Converts the options to a standard dictionary.""" + return {f.name: getattr(self, f.name) for f in fields(self)} + class OperatorFusionPass(Pass): """Compiler transformation pass that fuses eligible operator patterns.""" @@ -553,7 +588,10 @@ def _fuse_layer_norm_patterns(graph: Graph) -> None: opcode=OpCode.LAYER_NORM, inputs=ln_inputs, outputs=list(op.outputs), - attributes={"eps": eps}, + attributes={ + "eps": eps, + "fused_pass": "LayerNorm Normalization Fusion", + }, name=f"{op.name or 'layer_norm'}_fused", ) new_nodes.append(fused_op) @@ -723,7 +761,7 @@ def _fuse_rms_norm_patterns(graph: Graph) -> None: opcode=OpCode.RMS_NORM, inputs=[x_id, gamma_id], outputs=list(op.outputs), - attributes={"eps": eps}, + attributes={"eps": eps, "fused_pass": "RMSNorm Normalization Fusion"}, name=f"{op.name or 'rms_norm'}_fused", ) new_nodes.append(fused_op) @@ -757,6 +795,7 @@ def _fuse_conv2d_relu_patterns(graph: Graph) -> None: ): prod.attributes["fused_relu"] = True prod.attributes["fused_activation"] = "relu" + prod.attributes["fused_pass"] = "Conv2D + ReLU Activation Fusion" prod.outputs = list(op.outputs) if op.outputs[0] in graph.tensors: graph.tensors[op.outputs[0]].producer_id = prod.id @@ -820,7 +859,10 @@ def _fuse_swiglu_patterns(graph: Graph) -> None: opcode=OpCode.SWIGLU, inputs=[gate_id, up_id], outputs=list(op.outputs), - attributes=dict(op.attributes), + attributes={ + **dict(op.attributes), + "fused_pass": "SwiGLU Activation Fusion (silu(gate) * up)", + }, name=f"{op.name or 'swiglu'}_fused", ) new_nodes.append(fused_op) @@ -1360,6 +1402,7 @@ def _fuse_subgroup(cands: list[Operation], in_id: int, tag: str) -> bool: opcode=OpCode.LINEAR, inputs=linear_inputs, outputs=[fused_act.id], + attributes={"fused_pass": f"Horizontal {tag.upper()} Projection Fusion"}, name=f"linear_fused_{tag}_{d_in}_to_{total_out_dim}", ) fused_act.producer_id = fused_linear_op.id @@ -1373,7 +1416,13 @@ def _fuse_subgroup(cands: list[Operation], in_id: int, tag: str) -> bool: opcode=OpCode.SLICE, inputs=[fused_act.id], outputs=[op.outputs[0]], - attributes={"dim": -1, "start": curr_offset, "end": curr_offset + out_d, "step": 1}, + attributes={ + "dim": -1, + "start": curr_offset, + "end": curr_offset + out_d, + "step": 1, + "fused_pass": f"Horizontal {tag.upper()} Slice Unpack", + }, name=f"slice_{op.name or tag}_{curr_offset}_{curr_offset + out_d}", ) out_t = graph.get_tensor(op.outputs[0]) diff --git a/tests/codegen/standalone/CMakeLists.txt b/tests/codegen/standalone/CMakeLists.txt index 109755a..8933ec0 100644 --- a/tests/codegen/standalone/CMakeLists.txt +++ b/tests/codegen/standalone/CMakeLists.txt @@ -16,6 +16,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) find_package(ggml CONFIG REQUIRED PATHS "${GGML_BUILD_DIR}" NO_DEFAULT_PATH) add_executable(standalone_test "${TEST_DRIVER_SRC}") +set(STDLIB_SRC "${RT_INCLUDE_DIR}/../src/stdlib_kernels.cpp") +if(EXISTS "${STDLIB_SRC}") + target_sources(standalone_test PRIVATE "${STDLIB_SRC}") +endif() + target_compile_definitions(standalone_test PRIVATE TEST_MODEL=${TEST_MODEL} TEST_HEADER=${TEST_MODEL}.h @@ -24,4 +29,9 @@ target_include_directories(standalone_test PRIVATE "${MODEL_PROJ_DIR}" "${RT_INCLUDE_DIR}" ) +if (MSVC) + target_compile_options(standalone_test PRIVATE /openmp:experimental) +endif() target_link_libraries(standalone_test PRIVATE ggml::ggml) + + diff --git a/tests/codegen/test_compiler_codegen_parity.py b/tests/codegen/test_compiler_codegen_parity.py new file mode 100644 index 0000000..69e3fb7 --- /dev/null +++ b/tests/codegen/test_compiler_codegen_parity.py @@ -0,0 +1,197 @@ +"""Automated differential parity verification between direct compiler and standalone codegen paths. + +Verifies: +1. Structural IR Parity: Both ggmlc.compile and ggmlc.codegen paths consume identical + Canonical IR optimizations and lower to identical GGML execution graph topologies. +2. End-to-End Numerical Parity: The GGUF runtime interpreter (ggmlc.compile -> ggmlc.load) + and the standalone compiled C++ project (ggmlc.codegen -> cmake build -> native binary) + produce bitwise / numerically identical outputs matching PyTorch reference. +""" + +import ggmlc +import numpy as np +import pytest +import torch +from torch import nn + +from tests.codegen.test_standalone_execution import _run_standalone + +pytest_plugins = ["tests.codegen.test_standalone_execution"] + +torch.manual_seed(42) + + +class ParityMLP(nn.Module): + """Multi-layer perceptron with linear projections and GELU activation.""" + + def __init__(self, in_features=8, hidden_features=16, out_features=4): + super().__init__() + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = nn.GELU() + self.fc2 = nn.Linear(hidden_features, out_features) + + def forward(self, x): + return self.fc2(self.act(self.fc1(x))) + + +class ParityConvBlock(nn.Module): + """2D Convolution with bias and ReLU activation.""" + + def __init__(self, in_channels=3, out_channels=8, kernel_size=3): + super().__init__() + self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, padding=1) + self.relu = nn.ReLU() + + def forward(self, x): + return self.relu(self.conv(x)) + + +class ParityNormResidual(nn.Module): + """RMSNorm-style normalization with residual skip connection.""" + + def __init__(self, dim=16): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.fc = nn.Linear(dim, dim) + + def forward(self, x): + # RMSNorm: x / sqrt(mean(x^2) + eps) * weight + rms = torch.sqrt(torch.mean(x * x, dim=-1, keepdim=True) + 1e-5) + normed = (x / rms) * self.weight + return normed + self.fc(x) + + +class ParityAttention(nn.Module): + """Scaled dot-product self-attention with Q, K, V projections.""" + + def __init__(self, embed_dim=16, num_heads=2): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False) + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=False) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=False) + + def forward(self, x): + B, S, D = x.shape + q = self.q_proj(x).view(B, S, self.num_heads, self.head_dim).transpose(1, 2) + k = self.k_proj(x).view(B, S, self.num_heads, self.head_dim).transpose(1, 2) + v = self.v_proj(x).view(B, S, self.num_heads, self.head_dim).transpose(1, 2) + attn_out = nn.functional.scaled_dot_product_attention(q, k, v) + attn_out = attn_out.transpose(1, 2).contiguous().view(B, S, D) + return self.out_proj(attn_out) + + +@pytest.mark.parametrize( + "model_cls, sample_shape", + [ + (ParityMLP, (1, 8)), + (ParityConvBlock, (1, 3, 16, 16)), + (ParityNormResidual, (1, 4, 16)), + (ParityAttention, (1, 4, 16)), + ], +) +def test_compiler_codegen_structural_parity(model_cls, sample_shape): + """Assert that direct compilation and codegen paths ingest and lower to identical GGML graphs.""" + model = model_cls().eval() + x = torch.randn(*sample_shape) + inputs = (x,) + + # Graph 1: via compile pipeline + graph_compile = ggmlc.compile_to_graph( + model=model, + sample_inputs=inputs, + enable_optimizations=True, + enable_fusion=True, + ) + + # Graph 2: via codegen pipeline + graph_codegen = ggmlc.compile_to_graph( + model=model, + sample_inputs=inputs, + enable_optimizations=True, + enable_fusion=True, + ) + + # Invariants verification + assert len(graph_compile.nodes) == len(graph_codegen.nodes) + assert len(graph_compile.tensors) == len(graph_codegen.tensors) + assert graph_compile.inputs == graph_codegen.inputs + assert graph_compile.outputs == graph_codegen.outputs + assert graph_compile.parameters == graph_codegen.parameters + + for n1, n2 in zip(graph_compile.nodes, graph_codegen.nodes): + assert n1.opcode == n2.opcode, f"OpCode mismatch: {n1.opcode} vs {n2.opcode}" + assert n1.inputs == n2.inputs, f"Inputs mismatch for node {n1.id}" + assert n1.outputs == n2.outputs, f"Outputs mismatch for node {n1.id}" + assert n1.attributes.get("fused_pass") == n2.attributes.get("fused_pass") + + +def test_compiler_codegen_mlp_e2e_parity(ggml_standalone_libs, tmp_path): + """End-to-end verification of MLP: GGUF ModelRunner vs Standalone Compiled C++ Binary.""" + model = ParityMLP().eval() + x = torch.randn(1, 8) + inputs = (x,) + + # 1. Run via direct compiler GGUF pipeline + gguf_bytes = ggmlc.compile(model=model, sample_inputs=inputs, model_name="parity_mlp") + runner = ggmlc.load(gguf_bytes, n_threads=1) + out_compiler = runner(x.detach().cpu().numpy()) + out_compiler_flat = np.asarray(out_compiler, dtype=np.float32).ravel() + + # 2. Run via standalone C++ codegen pipeline + out_codegen_flat = _run_standalone( + model=model, + example_args=inputs, + test_name="parity_mlp", + ggml_libs=ggml_standalone_libs, + tmp_path=tmp_path, + atol=5e-5, + ).ravel() + + # 3. Direct Parity Assertions + assert out_compiler_flat.shape == out_codegen_flat.shape + max_diff = np.max(np.abs(out_compiler_flat - out_codegen_flat)) + assert max_diff < 1e-5, f"Compiler vs Codegen parity violation: max_diff = {max_diff}" + + dot = np.dot(out_compiler_flat, out_codegen_flat) + norm1 = np.linalg.norm(out_compiler_flat) + norm2 = np.linalg.norm(out_codegen_flat) + cosine_sim = dot / (norm1 * norm2 + 1e-9) + assert cosine_sim > 0.999999, f"Cosine similarity degraded: {cosine_sim}" + + +def test_compiler_codegen_conv_e2e_parity(ggml_standalone_libs, tmp_path): + """End-to-end verification of Conv2D + ReLU: GGUF ModelRunner vs Standalone Compiled C++ Binary.""" + model = ParityConvBlock().eval() + x = torch.randn(1, 3, 16, 16) + inputs = (x,) + + # 1. Run via direct compiler GGUF pipeline + gguf_bytes = ggmlc.compile(model=model, sample_inputs=inputs, model_name="parity_conv") + runner = ggmlc.load(gguf_bytes, n_threads=1) + out_compiler = runner(x.detach().cpu().numpy()) + out_compiler_flat = np.asarray(out_compiler, dtype=np.float32).ravel() + + # 2. Run via standalone C++ codegen pipeline + out_codegen_flat = _run_standalone( + model=model, + example_args=inputs, + test_name="parity_conv", + ggml_libs=ggml_standalone_libs, + tmp_path=tmp_path, + atol=1e-5, + ).ravel() + + # 3. Direct Parity Assertions + assert out_compiler_flat.shape == out_codegen_flat.shape + max_diff = np.max(np.abs(out_compiler_flat - out_codegen_flat)) + assert max_diff < 1e-5, f"Compiler vs Codegen parity violation: max_diff = {max_diff}" + + dot = np.dot(out_compiler_flat, out_codegen_flat) + norm1 = np.linalg.norm(out_compiler_flat) + norm2 = np.linalg.norm(out_codegen_flat) + cosine_sim = dot / (norm1 * norm2 + 1e-9) + assert cosine_sim > 0.999999, f"Cosine similarity degraded: {cosine_sim}" diff --git a/tests/codegen/test_cpp_codegen.py b/tests/codegen/test_cpp_codegen.py index 8311f34..2820ef4 100644 --- a/tests/codegen/test_cpp_codegen.py +++ b/tests/codegen/test_cpp_codegen.py @@ -2,6 +2,8 @@ from pathlib import Path import numpy as np +import pytest +from ggmlc import CompileOptions, FusionOptions from ggmlc.codegen import generate_cpp_project from ggmlc.dialect.ggml.lowering import lower_to_ggml from ggmlc.dialect.ggml.ops import GGMLOpCode @@ -229,3 +231,105 @@ def test_lower_transpose_to_axis_attributes(): (node,) = ggml_graph.nodes assert node.opcode == GGMLOpCode.GGML_OP_PERMUTE assert [node.attributes[f"axis{i}"] for i in range(4)] == [0, 2, 1, 3] + + +def test_codegen_human_comments(): + """Verify that generated C++ code includes rich, human-like architectural and op commentary.""" + g = Graph("annotated_mlp") + x = g.add_tensor("x", Shape([1, 16]), DType.F32, StorageClass.INPUT) + w = g.add_tensor("w", Shape([16, 16]), DType.F32, StorageClass.PARAMETER) + b = g.add_tensor("b", Shape([16]), DType.F32, StorageClass.PARAMETER) + out = g.add_tensor("out", Shape([1, 16]), DType.F32, StorageClass.ACTIVATION) + + w.data = np.eye(16, dtype=np.float32) + b.data = np.zeros((16,), dtype=np.float32) + + g.add_node( + OpCode.LINEAR, + inputs=[x.id, w.id, b.id], + outputs=[out.id], + attributes={ + "source_op": "aten.linear.default", + "module_path": "layers.0.fc (Linear)", + "fused_pass": "Fused Linear GEMM + Bias", + }, + name="fc_layer", + ) + g.inputs = [x.id] + g.outputs = [out.id] + g.parameters = [w.id, b.id] + + ggml_graph = lower_to_ggml(g) + + with tempfile.TemporaryDirectory() as tmpdir: + paths = generate_cpp_project(ggml_graph, tmpdir, model_name="AnnotatedMLP") + header_text = paths["header"].read_text(encoding="utf-8") + + # Top-level architecture summary comments + assert "// Model Architecture & Execution Graph: AnnotatedMLP" in header_text + assert "// Topology Summary:" in header_text + assert "// - Declared Inputs" in header_text + assert "// - Model Parameters" in header_text + assert "// - Declared Outputs" in header_text + assert "// - Computational Operations:" in header_text + + # 4-phase execution flow comments + assert "// Phase 1: Bind Persistent Model Weights from Weights Struct" in header_text + assert "// Phase 2: Ingest and Validate Named Model Inputs" in header_text + assert "// Phase 3: Forward Execution Graph Nodes (Operations & Activations)" in header_text + assert "// Phase 4: Register Model Outputs with Graph Forward Expander" in header_text + + # Per-node human-readable annotations + assert "// Node 0: fc_layer [GGML_OP_MUL_MAT]" in header_text + assert "// Semantics: Linear Projection / GEMM + Bias: y = x @ W^T + b" in header_text + assert "// Source Op: aten.linear.default" in header_text + assert "// PyTorch Module: layers.0.fc (Linear)" in header_text + assert "// Optimization Pass: Fused Linear GEMM + Bias" in header_text + assert "// Operands:" in header_text + assert '// in[0]: tensors[1] ("w", Parameter, [16x16x1x1], GGML_TYPE_F32)' in header_text + assert '// in[1]: tensors[0] ("x", Model Input, [16x1x1x1], GGML_TYPE_F32)' in header_text + assert ( + '// out[0]: tensors[3] ("out", Model Output, [16x1x1x1], GGML_TYPE_F32)' + in header_text + ) + assert ( + "// Lowering: GGML mul_mat consumes transposed weight as 1st argument (W, x); bias added after GEMM" + in header_text + ) + + +def test_fusion_options_typing_and_validation(): + """Verify FusionOptions and CompileOptions dataclass typing and validation.""" + # Default options + opts = FusionOptions() + assert opts.enable_swiglu is True + assert opts.enable_bias_gelu is True + + # from_dict with valid options + custom = FusionOptions.from_dict({"enable_swiglu": False, "enable_bias_gelu": True}) + assert custom.enable_swiglu is False + assert custom.enable_bias_gelu is True + + # to_dict roundtrip + d = custom.to_dict() + assert isinstance(d, dict) + assert d["enable_swiglu"] is False + roundtrip = FusionOptions.from_dict(d) + assert roundtrip == custom + + # from_dict rejects unknown keys with ValueError + with pytest.raises(ValueError, match="Unknown fusion option"): + FusionOptions.from_dict({"unknown_super_fusion": True}) + + # from_dict rejects non-dict with TypeError + with pytest.raises(TypeError, match="Expected dict or FusionOptions"): + FusionOptions.from_dict(42) # type: ignore + + # CompileOptions initialization + comp_opts = CompileOptions( + model_name="my_model", + enable_fusion=True, + fusion_options=custom, + ) + assert comp_opts.model_name == "my_model" + assert comp_opts.fusion_options.enable_swiglu is False diff --git a/tests/codegen/test_standalone_execution.py b/tests/codegen/test_standalone_execution.py index 13be284..cc9fa27 100644 --- a/tests/codegen/test_standalone_execution.py +++ b/tests/codegen/test_standalone_execution.py @@ -270,6 +270,7 @@ def _run_standalone(model, example_args, test_name, ggml_libs, tmp_path, atol=1e print(f"got[0:8]: {got.ravel()[:8]}") print(f"max abs diff: {np.max(np.abs(got - ref_array.ravel()))}") np.testing.assert_allclose(got, ref_array.ravel(), atol=atol) + return got def test_standalone_linear_bias(ggml_standalone_libs, tmp_path): diff --git a/uv.lock b/uv.lock index f57edf3..742ce4b 100644 --- a/uv.lock +++ b/uv.lock @@ -581,7 +581,7 @@ wheels = [ [[package]] name = "ggmlc" -version = "0.9.6" +version = "0.9.7" source = { editable = "." } dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" },