Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "scikit_build_core.build"

[project]
name = "ggmlc"
version = "0.9.6"
version = "0.9.7"
description = "A multi-framework neural network compiler lowering PyTorch, JAX, Flax, and Keras models to portable, high-performance GGML execution"
authors = [
{ name = "M. Yusuf Sarıgöz", email = "yusufsarigoz@gmail.com" },
Expand Down
9 changes: 8 additions & 1 deletion python/ggmlc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@
__version__ = "0.1.0"

__all__ = [
"CompileOptions",
"DType",
"FusionOptions",
"Graph",
"ModelRunner",
"OpCode",
Expand All @@ -48,6 +50,7 @@
"codegen",
"compile",
"compile_to_bytes",
"compile_to_graph",
"generate_cpp_project",
"get_available_devices",
"graph_to_mermaid",
Expand All @@ -58,10 +61,14 @@

def __getattr__(name: str) -> Any:
"""Lazy-loads compiler, IR, codegen, and visualization modules on demand."""
if name in ("compile", "compile_to_bytes", "codegen"):
if name in ("compile", "compile_to_bytes", "compile_to_graph", "codegen", "CompileOptions"):
import ggmlc.compiler as compiler_mod

return getattr(compiler_mod, name)
if name == "FusionOptions":
from ggmlc.transforms.fusion import FusionOptions

return FusionOptions
if name == "generate_cpp_project":
import ggmlc.codegen as codegen_mod

Expand Down
551 changes: 506 additions & 45 deletions python/ggmlc/codegen/cpp.py

Large diffs are not rendered by default.

253 changes: 130 additions & 123 deletions python/ggmlc/compiler.py

Large diffs are not rendered by default.

15 changes: 15 additions & 0 deletions python/ggmlc/frontend/pytorch/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -763,6 +763,21 @@ def _get_or_create_tensor(arg, default_name):
# Collect input tensor IDs and attributes
input_tensor_ids: list[int] = []
attributes: dict[str, Any] = {}
if hasattr(node.target, "__name__"):
attributes["source_op"] = f"aten.{node.target.__name__}"
else:
attributes["source_op"] = str(node.target)

if "nn_module_stack" in node.meta:
stack = node.meta["nn_module_stack"]
if isinstance(stack, dict) and stack:
mod_key = list(stack.keys())[-1]
mod_val = list(stack.values())[-1]
cls_name = ""
if isinstance(mod_val, tuple) and len(mod_val) >= 2:
cls_obj = mod_val[1]
cls_name = cls_obj.__name__ if hasattr(cls_obj, "__name__") else str(cls_obj)
attributes["module_path"] = f"{mod_key} ({cls_name})" if cls_name else str(mod_key)

if opcode == OpCode.TRANSPOSE:
# aten.transpose.int(self, dim0, dim1) or aten.t(self)
Expand Down
59 changes: 54 additions & 5 deletions python/ggmlc/transforms/fusion.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
import os
import re
import tempfile
from dataclasses import dataclass
from dataclasses import dataclass, fields
from typing import Any

import numpy as np

Expand Down Expand Up @@ -67,6 +68,40 @@ class FusionOptions:
# stock CUDA fused kernels stay intact.
enable_bake_affine: bool = True

@classmethod
def from_dict(cls, d: dict[str, Any] | FusionOptions | None) -> FusionOptions:
"""Constructs and validates a FusionOptions instance.

Args:
d: Dictionary of option name to boolean, FusionOptions instance, or None.

Returns:
Validated FusionOptions instance.

Raises:
ValueError: If unrecognized fusion option keys are provided.
TypeError: If input is not a dict or FusionOptions.
"""
if d is None:
return cls()
if isinstance(d, cls):
return d
if not isinstance(d, dict):
raise TypeError(f"Expected dict or FusionOptions, got {type(d).__name__}")

valid_fields = {f.name for f in fields(cls)}
unknown = set(d.keys()) - valid_fields
if unknown:
raise ValueError(
f"Unknown fusion option(s): {sorted(unknown)}. "
f"Valid options are: {sorted(valid_fields)}"
)
return cls(**{k: bool(v) for k, v in d.items()})

def to_dict(self) -> dict[str, bool]:
"""Converts the options to a standard dictionary."""
return {f.name: getattr(self, f.name) for f in fields(self)}


class OperatorFusionPass(Pass):
"""Compiler transformation pass that fuses eligible operator patterns."""
Expand Down Expand Up @@ -553,7 +588,10 @@ def _fuse_layer_norm_patterns(graph: Graph) -> None:
opcode=OpCode.LAYER_NORM,
inputs=ln_inputs,
outputs=list(op.outputs),
attributes={"eps": eps},
attributes={
"eps": eps,
"fused_pass": "LayerNorm Normalization Fusion",
},
name=f"{op.name or 'layer_norm'}_fused",
)
new_nodes.append(fused_op)
Expand Down Expand Up @@ -723,7 +761,7 @@ def _fuse_rms_norm_patterns(graph: Graph) -> None:
opcode=OpCode.RMS_NORM,
inputs=[x_id, gamma_id],
outputs=list(op.outputs),
attributes={"eps": eps},
attributes={"eps": eps, "fused_pass": "RMSNorm Normalization Fusion"},
name=f"{op.name or 'rms_norm'}_fused",
)
new_nodes.append(fused_op)
Expand Down Expand Up @@ -757,6 +795,7 @@ def _fuse_conv2d_relu_patterns(graph: Graph) -> None:
):
prod.attributes["fused_relu"] = True
prod.attributes["fused_activation"] = "relu"
prod.attributes["fused_pass"] = "Conv2D + ReLU Activation Fusion"
prod.outputs = list(op.outputs)
if op.outputs[0] in graph.tensors:
graph.tensors[op.outputs[0]].producer_id = prod.id
Expand Down Expand Up @@ -820,7 +859,10 @@ def _fuse_swiglu_patterns(graph: Graph) -> None:
opcode=OpCode.SWIGLU,
inputs=[gate_id, up_id],
outputs=list(op.outputs),
attributes=dict(op.attributes),
attributes={
**dict(op.attributes),
"fused_pass": "SwiGLU Activation Fusion (silu(gate) * up)",
},
name=f"{op.name or 'swiglu'}_fused",
)
new_nodes.append(fused_op)
Expand Down Expand Up @@ -1360,6 +1402,7 @@ def _fuse_subgroup(cands: list[Operation], in_id: int, tag: str) -> bool:
opcode=OpCode.LINEAR,
inputs=linear_inputs,
outputs=[fused_act.id],
attributes={"fused_pass": f"Horizontal {tag.upper()} Projection Fusion"},
name=f"linear_fused_{tag}_{d_in}_to_{total_out_dim}",
)
fused_act.producer_id = fused_linear_op.id
Expand All @@ -1373,7 +1416,13 @@ def _fuse_subgroup(cands: list[Operation], in_id: int, tag: str) -> bool:
opcode=OpCode.SLICE,
inputs=[fused_act.id],
outputs=[op.outputs[0]],
attributes={"dim": -1, "start": curr_offset, "end": curr_offset + out_d, "step": 1},
attributes={
"dim": -1,
"start": curr_offset,
"end": curr_offset + out_d,
"step": 1,
"fused_pass": f"Horizontal {tag.upper()} Slice Unpack",
},
name=f"slice_{op.name or tag}_{curr_offset}_{curr_offset + out_d}",
)
out_t = graph.get_tensor(op.outputs[0])
Expand Down
10 changes: 10 additions & 0 deletions tests/codegen/standalone/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
find_package(ggml CONFIG REQUIRED PATHS "${GGML_BUILD_DIR}" NO_DEFAULT_PATH)

add_executable(standalone_test "${TEST_DRIVER_SRC}")
set(STDLIB_SRC "${RT_INCLUDE_DIR}/../src/stdlib_kernels.cpp")
if(EXISTS "${STDLIB_SRC}")
target_sources(standalone_test PRIVATE "${STDLIB_SRC}")
endif()

target_compile_definitions(standalone_test PRIVATE
TEST_MODEL=${TEST_MODEL}
TEST_HEADER=${TEST_MODEL}.h
Expand All @@ -24,4 +29,9 @@ target_include_directories(standalone_test PRIVATE
"${MODEL_PROJ_DIR}"
"${RT_INCLUDE_DIR}"
)
if (MSVC)
target_compile_options(standalone_test PRIVATE /openmp:experimental)
endif()
target_link_libraries(standalone_test PRIVATE ggml::ggml)


197 changes: 197 additions & 0 deletions tests/codegen/test_compiler_codegen_parity.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,197 @@
"""Automated differential parity verification between direct compiler and standalone codegen paths.

Verifies:
1. Structural IR Parity: Both ggmlc.compile and ggmlc.codegen paths consume identical
Canonical IR optimizations and lower to identical GGML execution graph topologies.
2. End-to-End Numerical Parity: The GGUF runtime interpreter (ggmlc.compile -> ggmlc.load)
and the standalone compiled C++ project (ggmlc.codegen -> cmake build -> native binary)
produce bitwise / numerically identical outputs matching PyTorch reference.
"""

import ggmlc
import numpy as np
import pytest
import torch
from torch import nn

from tests.codegen.test_standalone_execution import _run_standalone

pytest_plugins = ["tests.codegen.test_standalone_execution"]

torch.manual_seed(42)


class ParityMLP(nn.Module):
"""Multi-layer perceptron with linear projections and GELU activation."""

def __init__(self, in_features=8, hidden_features=16, out_features=4):
super().__init__()
self.fc1 = nn.Linear(in_features, hidden_features)
self.act = nn.GELU()
self.fc2 = nn.Linear(hidden_features, out_features)

def forward(self, x):
return self.fc2(self.act(self.fc1(x)))


class ParityConvBlock(nn.Module):
"""2D Convolution with bias and ReLU activation."""

def __init__(self, in_channels=3, out_channels=8, kernel_size=3):
super().__init__()
self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, padding=1)
self.relu = nn.ReLU()

def forward(self, x):
return self.relu(self.conv(x))


class ParityNormResidual(nn.Module):
"""RMSNorm-style normalization with residual skip connection."""

def __init__(self, dim=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.fc = nn.Linear(dim, dim)

def forward(self, x):
# RMSNorm: x / sqrt(mean(x^2) + eps) * weight
rms = torch.sqrt(torch.mean(x * x, dim=-1, keepdim=True) + 1e-5)
normed = (x / rms) * self.weight
return normed + self.fc(x)


class ParityAttention(nn.Module):
"""Scaled dot-product self-attention with Q, K, V projections."""

def __init__(self, embed_dim=16, num_heads=2):
super().__init__()
self.embed_dim = embed_dim
self.num_heads = num_heads
self.head_dim = embed_dim // num_heads
self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False)
self.v_proj = nn.Linear(embed_dim, embed_dim, bias=False)
self.out_proj = nn.Linear(embed_dim, embed_dim, bias=False)

def forward(self, x):
B, S, D = x.shape
q = self.q_proj(x).view(B, S, self.num_heads, self.head_dim).transpose(1, 2)
k = self.k_proj(x).view(B, S, self.num_heads, self.head_dim).transpose(1, 2)
v = self.v_proj(x).view(B, S, self.num_heads, self.head_dim).transpose(1, 2)
attn_out = nn.functional.scaled_dot_product_attention(q, k, v)
attn_out = attn_out.transpose(1, 2).contiguous().view(B, S, D)
return self.out_proj(attn_out)


@pytest.mark.parametrize(
"model_cls, sample_shape",
[
(ParityMLP, (1, 8)),
(ParityConvBlock, (1, 3, 16, 16)),
(ParityNormResidual, (1, 4, 16)),
(ParityAttention, (1, 4, 16)),
],
)
def test_compiler_codegen_structural_parity(model_cls, sample_shape):
"""Assert that direct compilation and codegen paths ingest and lower to identical GGML graphs."""
model = model_cls().eval()
x = torch.randn(*sample_shape)
inputs = (x,)

# Graph 1: via compile pipeline
graph_compile = ggmlc.compile_to_graph(
model=model,
sample_inputs=inputs,
enable_optimizations=True,
enable_fusion=True,
)

# Graph 2: via codegen pipeline
graph_codegen = ggmlc.compile_to_graph(
model=model,
sample_inputs=inputs,
enable_optimizations=True,
enable_fusion=True,
)

# Invariants verification
assert len(graph_compile.nodes) == len(graph_codegen.nodes)
assert len(graph_compile.tensors) == len(graph_codegen.tensors)
assert graph_compile.inputs == graph_codegen.inputs
assert graph_compile.outputs == graph_codegen.outputs
assert graph_compile.parameters == graph_codegen.parameters

for n1, n2 in zip(graph_compile.nodes, graph_codegen.nodes):
assert n1.opcode == n2.opcode, f"OpCode mismatch: {n1.opcode} vs {n2.opcode}"
assert n1.inputs == n2.inputs, f"Inputs mismatch for node {n1.id}"
assert n1.outputs == n2.outputs, f"Outputs mismatch for node {n1.id}"
assert n1.attributes.get("fused_pass") == n2.attributes.get("fused_pass")


def test_compiler_codegen_mlp_e2e_parity(ggml_standalone_libs, tmp_path):
"""End-to-end verification of MLP: GGUF ModelRunner vs Standalone Compiled C++ Binary."""
model = ParityMLP().eval()
x = torch.randn(1, 8)
inputs = (x,)

# 1. Run via direct compiler GGUF pipeline
gguf_bytes = ggmlc.compile(model=model, sample_inputs=inputs, model_name="parity_mlp")
runner = ggmlc.load(gguf_bytes, n_threads=1)
out_compiler = runner(x.detach().cpu().numpy())
out_compiler_flat = np.asarray(out_compiler, dtype=np.float32).ravel()

# 2. Run via standalone C++ codegen pipeline
out_codegen_flat = _run_standalone(
model=model,
example_args=inputs,
test_name="parity_mlp",
ggml_libs=ggml_standalone_libs,
tmp_path=tmp_path,
atol=5e-5,
).ravel()

# 3. Direct Parity Assertions
assert out_compiler_flat.shape == out_codegen_flat.shape
max_diff = np.max(np.abs(out_compiler_flat - out_codegen_flat))
assert max_diff < 1e-5, f"Compiler vs Codegen parity violation: max_diff = {max_diff}"

dot = np.dot(out_compiler_flat, out_codegen_flat)
norm1 = np.linalg.norm(out_compiler_flat)
norm2 = np.linalg.norm(out_codegen_flat)
cosine_sim = dot / (norm1 * norm2 + 1e-9)
assert cosine_sim > 0.999999, f"Cosine similarity degraded: {cosine_sim}"


def test_compiler_codegen_conv_e2e_parity(ggml_standalone_libs, tmp_path):
"""End-to-end verification of Conv2D + ReLU: GGUF ModelRunner vs Standalone Compiled C++ Binary."""
model = ParityConvBlock().eval()
x = torch.randn(1, 3, 16, 16)
inputs = (x,)

# 1. Run via direct compiler GGUF pipeline
gguf_bytes = ggmlc.compile(model=model, sample_inputs=inputs, model_name="parity_conv")
runner = ggmlc.load(gguf_bytes, n_threads=1)
out_compiler = runner(x.detach().cpu().numpy())
out_compiler_flat = np.asarray(out_compiler, dtype=np.float32).ravel()

# 2. Run via standalone C++ codegen pipeline
out_codegen_flat = _run_standalone(
model=model,
example_args=inputs,
test_name="parity_conv",
ggml_libs=ggml_standalone_libs,
tmp_path=tmp_path,
atol=1e-5,
).ravel()

# 3. Direct Parity Assertions
assert out_compiler_flat.shape == out_codegen_flat.shape
max_diff = np.max(np.abs(out_compiler_flat - out_codegen_flat))
assert max_diff < 1e-5, f"Compiler vs Codegen parity violation: max_diff = {max_diff}"

dot = np.dot(out_compiler_flat, out_codegen_flat)
norm1 = np.linalg.norm(out_compiler_flat)
norm2 = np.linalg.norm(out_codegen_flat)
cosine_sim = dot / (norm1 * norm2 + 1e-9)
assert cosine_sim > 0.999999, f"Cosine similarity degraded: {cosine_sim}"
Loading
Loading