Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions examples/models/openclip_model.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
"""Real checkpoint loaders for OpenCLIP models"""

from __future__ import annotations

import torch
from ggmlc.pipeline import VisionPreprocessor
from open_clip import create_model_from_pretrained
from open_clip.transform import MaybeConvertMode, MaybeToTensor
from timm.utils import reparameterize_model
from torchvision.transforms.transforms import CenterCrop, Normalize, Resize

__all__ = [
"load_openclip_image_encoder_model",
]

# preprocess
# Compose(
# Resize(size=256, interpolation=bicubic, max_size=None, antialias=True)
# CenterCrop(size=(256, 256))
# MaybeConvertMode()
# MaybeToTensor()
# Normalize(mean=(0.48145466, 0.4578275, 0.40821073),
# std=(0.26862954, 0.26130258, 0.27577711))
# )

def make_vision_preprocessor(preprocess) -> VisionPreprocessor:
"""Creates a GGMLC VisionPreprocessor."""
kwargs = {}
for transform in preprocess.transforms:
if isinstance(transform, Resize):
target_size = (transform.size, transform.size) \
if isinstance(transform.size, int) else transform.size
kwargs["target_size"] = target_size
elif isinstance(transform, Normalize):
kwargs["mean"] = transform.mean
kwargs["std"] = transform.std
elif isinstance(transform, (CenterCrop, MaybeConvertMode, MaybeToTensor)):
continue # Ignore
else:
raise TypeError(f"Unsupported transform: {transform}")
return VisionPreprocessor(**kwargs)


def load_openclip_image_encoder_model(
variant: str = "hf-hub:timm/MobileCLIP2-S2-OpenCLIP",
with_preprocessor: bool = True,
**kwargs
) -> tuple[torch.nn.Module, tuple[torch.Tensor, ...], list[str], VisionPreprocessor]:
"""Loads OpenCLIP pretrained image encoder model.
"""

if "return_transform" in kwargs:
raise ValueError("return_transform is not supported; use `with_preprocessor` instead")

# Load the model and optionally preprocessing transforms
create_result = create_model_from_pretrained(
variant, return_transform=with_preprocessor, **kwargs)
model = create_result if not with_preprocessor else create_result[0]

# Prepare for inference/model exporting purposes
model = reparameterize_model(model.visual.eval())

# Create example input for the model
image_size = model.image_size
pixel_values = torch.randn(1, 3, image_size[0], image_size[1], dtype=torch.float32)
example_input = (pixel_values,)
input_names = ["pixel_values"]

# Create a wrapper module for the image encoder
class OpenCLIPImageEncoderWrapper(torch.nn.Module):
def __init__(self, base):
super().__init__()
self.base = base
def forward(self, pixel_values):
return torch.nn.functional.normalize(self.base(pixel_values), dim=-1)
wrapped_model = OpenCLIPImageEncoderWrapper(model)

if with_preprocessor:
# Create a VisionPreprocessor for the model
preprocess = create_result[1]
vision_preprocessor = make_vision_preprocessor(preprocess)
return wrapped_model, example_input, input_names, vision_preprocessor

return wrapped_model, example_input, input_names
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ dev = [
"sphinx-copybutton>=0.5.0",
"mermaidx>=0.9.0",
"laya>=0.3.4",
"open_clip_torch>=3.3.0",
]

[project.scripts]
Expand Down
200 changes: 179 additions & 21 deletions python/ggmlc/codegen/cpp.py

Large diffs are not rendered by default.

52 changes: 49 additions & 3 deletions python/ggmlc/dialect/ggml/lowering.py
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,11 @@ def _lower_op(
in_t = c_graph.get_tensor(in_ids[0])
R = len(in_t.shape.dims)
dim = attrs.get("dim", -1)
if isinstance(dim, (tuple, list)):
raise NotImplementedError(
f"Multi-axis MEAN '{op.name}' must be chained into single-axis "
f"ops by the frontend, got dim={dim}"
)
if dim < 0:
dim += R
ggml_dim = R - 1 - dim
Expand All @@ -287,6 +292,11 @@ def _lower_op(
R = len(in_t.shape.dims)
axes = attrs.get("axes", None) or attrs.get("dim", -1)
if isinstance(axes, (tuple, list)):
if len(axes) > 1:
raise NotImplementedError(
f"Multi-axis {opcode.value} '{op.name}' must be chained into "
f"single-axis ops by the frontend, got axes={axes}"
)
dim = axes[0] if len(axes) > 0 else -1
else:
dim = int(axes)
Expand All @@ -313,12 +323,18 @@ def _lower_op(
op.name,
)
elif opcode == OpCode.GELU:
approx = str(attrs.get("approximate", "none")).lower()
gelu_op = (
GGMLUnaryOpCode.GGML_UNARY_OP_GELU_ERF
if approx in ("none", "erf")
else GGMLUnaryOpCode.GGML_UNARY_OP_GELU
)
return GGMLOpDef(
op.id,
GGMLOpCode.GGML_OP_UNARY,
in_ids,
out_ids,
{"unary_op": int(GGMLUnaryOpCode.GGML_UNARY_OP_GELU)},
{"unary_op": int(gelu_op)},
op.name,
)
elif opcode == OpCode.SILU:
Expand Down Expand Up @@ -514,7 +530,21 @@ def _lower_op(
):
w_ic = w_t.shape.dims[1].evaluate({})
x_ic = x_t.shape.dims[1].evaluate({})
if w_ic == 1 and (groups > 1 or x_ic > 1):
w_oc = w_t.shape.dims[0].evaluate({}) if w_t.shape.dims[0].is_static() else None
if w_ic == 1 and groups > 1:
# True depthwise needs one filter per input channel.
# Grouped-with-multiplier (e.g. groups=80, C_in=80,
# C_out=160) has no ggml op and must be decomposed
# per group by the importer.
if (w_oc is None or w_oc == groups) and x_ic == groups:
is_dw = True
else:
raise ValueError(
f"grouped convolution '{op.name}' (id={op.id}) is not "
f"depthwise (groups={groups}, in_channels={x_ic}, "
f"out_channels={w_oc}); ggml has no grouped-conv op"
)
elif w_ic == 1 and x_ic > 1:
is_dw = True
elif len(w_t.shape.dims) == 3 and w_t.shape.dims[1].is_static():
# Conv1d depthwise: weight [C, 1, K]
Expand Down Expand Up @@ -568,6 +598,20 @@ def _lower_op(
if dim < 0:
dim += R
ggml_dim = R - 1 - dim if R > 0 else 0
if ggml_dim > 3:
# Outer dims fold into ne[3] (canonical_shape_to_ggml_ne): the
# slice lands on ggml dim 3, with an offset multiplier covering
# the folded dims inner to the sliced one.
ggml_dim = 3
mult = 1
for d in in_t.shape.dims[dim + 1 : R - 3]:
if isinstance(d, StaticDim):
mult *= d.value
else:
raise ValueError( # noqa: TRY004 - unsupported shape, not a type error
f"slice of folded dim {dim} needs static inner shape (tensor '{in_t.name}')"
)
attrs["offset_mult"] = int(mult)
attrs["ggml_dim"] = int(ggml_dim)
attrs["start"] = int(start)
return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_VIEW, in_ids, out_ids, attrs, op.name)
Expand All @@ -592,7 +636,9 @@ def _lower_op(
else:
axes[i] = i
if any(ax >= 4 for ax in axes):
return GGMLOpDef(op.id, GGMLOpCode.GGML_OP_RESHAPE, in_ids, out_ids, attrs, op.name)
raise NotImplementedError(
f"Permute {op.name} results in a non-expressible ggml permute: axes={axes}"
)
attrs["axis0"] = axes[0]
attrs["axis1"] = axes[1]
attrs["axis2"] = axes[2]
Expand Down
1 change: 1 addition & 0 deletions python/ggmlc/dialect/ggml/ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ class GGMLUnaryOpCode(IntEnum):
GGML_UNARY_OP_HARDSWISH = 11
GGML_UNARY_OP_HARDSIGMOID = 12
GGML_UNARY_OP_EXP = 13
GGML_UNARY_OP_GELU_ERF = 16


@unique
Expand Down
97 changes: 94 additions & 3 deletions python/ggmlc/frontend/jax/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import numpy as np

from ggmlc.ir.dtype import DType
from ggmlc.ir.graph import Graph
from ggmlc.ir.graph import Graph, chain_single_axis_reduction
from ggmlc.ir.op import OpCode
from ggmlc.ir.shape import Shape, StaticDim
from ggmlc.ir.tensor import StorageClass, Tensor
Expand Down Expand Up @@ -66,6 +66,58 @@ def _jax_dtype_to_dtype(dtype: Any) -> DType:
return DType.from_numpy(dt)


def _pad_nhwc_asymmetric(
g: Graph,
t: Tensor,
h_pair: tuple[int, int],
w_pair: tuple[int, int],
) -> Tensor:
"""Zero-pad NHWC sides ggml conv padding cannot express (top/left/bottom/right).

Implemented as zero concats (fully supported in both runners); the caller
then uses the symmetric minimum as the conv padding.
"""
if t.dtype != DType.F32:
raise NotImplementedError(f"asymmetric pad needs F32 input, got {t.dtype}")
# NOTE: keep names free of brackets/colons: parameter names are matched
# verbatim by the GGUF loader, which chokes on jaxpr-style characters.
tag = f"padconv_{len(g.tensors)}"
dims = [int(d.value) if isinstance(d, StaticDim) else int(d) for d in t.shape.dims]
for axis, (before, after) in ((1, h_pair), (2, w_pair)):
for side, k in (("pre", before), ("post", after)):
if k <= 0:
continue
z_dims = list(dims)
z_dims[axis] = k
z = g.add_tensor(
name=f"{tag}_zeros_d{axis}_{side}",
shape=Shape.from_tuple(tuple(z_dims)),
dtype=t.dtype,
storage=StorageClass.CONSTANT,
data=np.zeros(tuple(z_dims), dtype=np.float32),
)
g.parameters.append(z.id)
padded_dims = list(dims)
padded_dims[axis] = dims[axis] + k
out = g.add_tensor(
name=f"{tag}_pad_d{axis}_{side}",
shape=Shape.from_tuple(tuple(padded_dims)),
dtype=t.dtype,
storage=StorageClass.ACTIVATION,
)
ins = [z.id, t.id] if side == "pre" else [t.id, z.id]
g.add_op(
opcode=OpCode.CONCAT,
inputs=ins,
outputs=[out.id],
attributes={"dim": axis},
name=f"{tag}_pad_d{axis}_{side}",
)
t = out
dims = padded_dims
return t


def _import_equations(
eqns: Sequence[Any],
g: Graph,
Expand Down Expand Up @@ -908,6 +960,21 @@ def _dim_val(d):
else:
lhs_t = g.get_tensor(in_tids[0])

# Asymmetric padding (TF 'SAME' with odd totals, explicit pads):
# ggml convs only take symmetric padding, so materialize the
# excess sides as zero concats and keep the symmetric minimum.
pt0, pt1 = int(padding[0][0]), int(padding[0][1])
pl0, pl1 = int(padding[1][0]), int(padding[1][1])
sym_h, sym_w = min(pt0, pt1), min(pl0, pl1)
if (pt0, pt1) != (sym_h, sym_h) or (pl0, pl1) != (sym_w, sym_w):
lhs_t = _pad_nhwc_asymmetric(
g,
lhs_t,
(pt0 - sym_h, pt1 - sym_h),
(pl0 - sym_w, pl1 - sym_w),
)
sym_padding = (sym_h, sym_w)

rhs_t = g.get_tensor(in_tids[1])

def _dim_val(d):
Expand Down Expand Up @@ -972,7 +1039,7 @@ def _dim_val(d):
)
conv_attrs = {
"stride": tuple(window_strides),
"padding": (padding[0][0], padding[1][0]),
"padding": sym_padding,
"dilation": tuple(rhs_dilation),
"groups": int(feature_group_count),
}
Expand All @@ -994,7 +1061,7 @@ def _dim_val(d):
else:
conv_attrs = {
"stride": tuple(window_strides),
"padding": (padding[0][0], padding[1][0]),
"padding": sym_padding,
"dilation": tuple(rhs_dilation),
"groups": int(feature_group_count),
}
Expand Down Expand Up @@ -1944,6 +2011,30 @@ def _dim_val(d):
f"Unhandled primitive '{prim_name}' reached op emission with None opcode."
)

if opcode in (OpCode.SUM, OpCode.AMAX, OpCode.AMIN):
# Multi-axis reductions (e.g. global average pool) chain into
# single-axis ops; ggml reduces one axis (JAX out drops the axes).
raw_axes = eqn.params.get("axes", (-1,))
if isinstance(raw_axes, (tuple, list)):
axes_list = [int(a) for a in raw_axes]
else:
axes_list = [int(raw_axes)]
in_t = g.get_tensor(in_tids[0])
rank = len(in_t.shape.dims)
normed = [a + rank if a < 0 else a for a in axes_list]
if len(normed) > 1:
chained = chain_single_axis_reduction(
g,
in_t,
opcode,
normed,
False,
f"{prim_name}_{out_var}",
final_out=out_t,
)
var_to_tensor[out_var] = chained
continue

g.add_op(
opcode=opcode,
inputs=in_tids,
Expand Down
Loading
Loading