diff --git a/src/chop/nn/quantizers/_minifloat_mx/__init__.py b/src/chop/nn/quantizers/_minifloat_mx/__init__.py index b25cbfae2..0ae67e0c7 100644 --- a/src/chop/nn/quantizers/_minifloat_mx/__init__.py +++ b/src/chop/nn/quantizers/_minifloat_mx/__init__.py @@ -5,7 +5,11 @@ """ from .meta import MinifloatMeta, MinifloatTensorMeta -from .fake import extract_minifloat_component, compose_minifloat_component +from .fake import ( + compose_minifloat_component, + extract_minifloat_component, + quantize_minifloat_value, +) from .minifloat import minifloat_quantizer_sim __all__ = [ @@ -13,5 +17,6 @@ "MinifloatTensorMeta", "extract_minifloat_component", "compose_minifloat_component", + "quantize_minifloat_value", "minifloat_quantizer_sim", ] diff --git a/src/chop/nn/quantizers/_minifloat_mx/fake.py b/src/chop/nn/quantizers/_minifloat_mx/fake.py index 0b2c85ae9..3ab95e7fe 100644 --- a/src/chop/nn/quantizers/_minifloat_mx/fake.py +++ b/src/chop/nn/quantizers/_minifloat_mx/fake.py @@ -8,89 +8,125 @@ from .meta import MinifloatMeta -def extract_minifloat_component(x: Tensor, minifloat_meta: MinifloatMeta) -> Tensor: - """ - Extract minifloat representation from float tensor. - - Args: - x: Input float tensor - minifloat_meta: Minifloat format specification - - Returns: - Tensor of uint16 containing minifloat representation - """ +def _minifloat_fields( + x: Tensor, minifloat_meta: MinifloatMeta +) -> tuple[Tensor, Tensor, Tensor]: + """Return (sign, biased exponent, fraction) for one minifloat rounding.""" y_exp_bits = minifloat_meta.exp_bits y_frac_bits = minifloat_meta.frac_bits always_finite = minifloat_meta.is_finite round_mode = minifloat_meta.round_mode - y_exp_bias = (1 << (y_exp_bits - 1)) - 1 + y_exp_bias = 1 if y_exp_bits == 1 else (1 << (y_exp_bits - 1)) - 1 y_exp_max = (1 << y_exp_bits) - 1 if always_finite else (1 << y_exp_bits) - 2 y_exp_max_biased = y_exp_max - y_exp_bias - y_exp_min = 0 - y_exp_min_biased = y_exp_min - y_exp_bias + y_exp_min_biased = 1 - y_exp_bias y_frac_max = (1 << y_frac_bits) - 1 + y_frac_levels = 1 << y_frac_bits x = x.to(torch.float32) - y_sign = x < 0 - x_int32 = x.abs().view(torch.int32) - flush_to_zero = (x_int32 & 0x7F800000) == 0 - x_normal = torch.where(flush_to_zero, 0.0, x) - - x_frac, x_exp = x_normal.abs().frexp() - x_frac = x_frac * 2 - x_exp = x_exp - 1 - - if not always_finite: - x_is_inf = x.isinf() - x_is_nan = x.isnan() - - y_exp = x_exp - underflow = y_exp < y_exp_min_biased - overflow = y_exp > y_exp_max_biased - y_exp = y_exp + y_exp_bias - - y_frac = x_frac.view(torch.int32) & 0x7FFFFF - - if round_mode == "rz": - y_frac = y_frac >> (23 - y_frac_bits) - else: - y_frac = (y_frac >> 8).float() - div = 1 << (15 - y_frac_bits) - y_frac = y_frac / div + y_sign = torch.signbit(x) + magnitude = x.abs() + nonzero_finite = torch.isfinite(magnitude) & (magnitude != 0) + + def round_magnitude(value: Tensor) -> Tensor: + if round_mode == "rn": + return value.round() + if round_mode == "rz": + return value.floor() if round_mode == "ru": - y_frac = y_frac.ceil() - elif round_mode == "rd": - y_frac = y_frac.floor() - elif round_mode == "rn": - y_frac = y_frac.round() - else: - raise ValueError(f"Unknown rounding mode: {round_mode}") - y_frac = y_frac.to(torch.int32) - - y_is_subnormal = (y_exp == y_exp_min) & (y_frac != 0) - y_frac = torch.where(y_is_subnormal, (y_frac | (1 << y_frac_bits)) >> 1, y_frac) - - # underflow -> 0 - y_frac = torch.where(underflow, 0, y_frac) - y_exp = torch.where(underflow, 0, y_exp) - # overflow -> max + return torch.where(y_sign, value.floor(), value.ceil()) + if round_mode == "rd": + return torch.where(y_sign, value.ceil(), value.floor()) + raise ValueError(f"Unknown rounding mode: {round_mode}") + + min_normal = float(2**y_exp_min_biased) + subnormal_step = min_normal / y_frac_levels + is_subnormal = nonzero_finite & (magnitude < min_normal) + + safe_magnitude = torch.where( + nonzero_finite, + magnitude, + torch.ones_like(magnitude), + ) + unbiased_exp = torch.floor(torch.log2(safe_magnitude)) + overflow = (~torch.isfinite(magnitude)) | ( + nonzero_finite & (unbiased_exp > y_exp_max_biased) + ) + clamped_exp = unbiased_exp.clamp( + min=y_exp_min_biased, + max=y_exp_max_biased, + ) + normal_fraction = (safe_magnitude / torch.exp2(clamped_exp) - 1.0) * y_frac_levels + y_frac = round_magnitude(normal_fraction).to(torch.int32) + y_exp = (clamped_exp + y_exp_bias).to(torch.int32) + + carry = y_frac >= y_frac_levels + y_frac = torch.where(carry, 0, y_frac) + y_exp = torch.where(carry, y_exp + 1, y_exp) + + subnormal_fraction = round_magnitude(magnitude / subnormal_step).to(torch.int32) + subnormal_carry = subnormal_fraction >= y_frac_levels + y_frac = torch.where( + is_subnormal, + torch.where(subnormal_carry, 0, subnormal_fraction), + y_frac, + ) + y_exp = torch.where( + is_subnormal, + torch.where(subnormal_carry, 1, 0), + y_exp, + ) + + overflow = overflow | (y_exp > y_exp_max) y_frac = torch.where(overflow, y_frac_max, y_frac) y_exp = torch.where(overflow, y_exp_max, y_exp) - # flush to zero - y_frac = torch.where(flush_to_zero, 0, y_frac) - y_exp = torch.where(flush_to_zero, 0, y_exp) + y_frac = torch.where(nonzero_finite | overflow, y_frac, 0) + y_exp = torch.where(nonzero_finite | overflow, y_exp, 0) + return y_sign, y_exp, y_frac - if not always_finite: - y_frac = torch.where(x_is_inf, 0, y_frac) - y_frac = torch.where(x_is_nan, (1 << y_frac_bits) - 1, y_frac) - y_exp = torch.where(x_is_inf, y_exp_max, y_exp) - y_exp = torch.where(x_is_nan, y_exp_max, y_exp) +def extract_minifloat_component(x: Tensor, minifloat_meta: MinifloatMeta) -> Tensor: + """ + Extract minifloat representation from float tensor. + + Args: + x: Input float tensor + minifloat_meta: Minifloat format specification + + Returns: + Tensor of uint16 containing minifloat representation + """ + y_sign, y_exp, y_frac = _minifloat_fields(x, minifloat_meta) + y_frac_bits = minifloat_meta.frac_bits y = (y_exp << y_frac_bits) | y_frac - y = torch.where(y_sign, y + (1 << (y_exp_bits + y_frac_bits)), y) - y = y.to(torch.uint16) - return y + y = torch.where(y_sign, y + (1 << (minifloat_meta.exp_bits + y_frac_bits)), y) + return y.to(torch.uint16) + + +def quantize_minifloat_value(x: Tensor, minifloat_meta: MinifloatMeta) -> Tensor: + """Round to the minifloat grid and return the value, skipping the encoding. + + Equivalent to ``compose_minifloat_component(extract_minifloat_component(x))`` + but without the uint16 round trip. The encoder never emits an inf/nan + exponent code, so the decoder's special-value branches cannot apply here. + """ + y_sign, y_exp, y_frac = _minifloat_fields(x, minifloat_meta) + y_exp_bias = ( + 1 + if minifloat_meta.exp_bits == 1 + else ((1 << (minifloat_meta.exp_bits - 1)) - 1) + ) + frac_levels = 1 << minifloat_meta.frac_bits + subnormal_step = float(2 ** (1 - y_exp_bias)) / frac_levels + fraction = y_frac.to(torch.float32) + magnitude = torch.where( + y_exp == 0, + fraction * subnormal_step, + (1.0 + fraction / frac_levels) + * torch.exp2((y_exp - y_exp_bias).to(torch.float32)), + ) + return torch.where(y_sign, -magnitude, magnitude) def compose_minifloat_component( @@ -115,13 +151,13 @@ def compose_minifloat_component( x_sign_mask = 1 << (exp_bits + frac_bits) x_frac_mask = (1 << frac_bits) - 1 - x_exp_bias = (1 << (exp_bits - 1)) - 1 + x_exp_bias = 1 if exp_bits == 1 else (1 << (exp_bits - 1)) - 1 assert elements.dtype == torch.uint16 elements = elements.to(torch.int32) y_sign = (elements & x_sign_mask) << (31 - (exp_bits + frac_bits)) - elements = elements & 0x7FFF + elements = elements & (x_sign_mask - 1) x_exp = (elements >> frac_bits) & ((1 << exp_bits) - 1) x_frac = elements & x_frac_mask is_subnormal = (x_exp == 0) & (x_frac != 0) diff --git a/src/chop/nn/quantizers/_minifloat_mx/meta.py b/src/chop/nn/quantizers/_minifloat_mx/meta.py index 94e48f9b8..b8645e9d7 100644 --- a/src/chop/nn/quantizers/_minifloat_mx/meta.py +++ b/src/chop/nn/quantizers/_minifloat_mx/meta.py @@ -15,7 +15,7 @@ class MinifloatMeta: Args: exp_bits: Number of exponent bits frac_bits: Number of fraction bits - is_finite: Whether the minifloat type supports inf/nan + is_finite: Whether every exponent code represents a finite value round_mode: Rounding mode - "rn" (nearest), "rd" (down), "ru" (up), "rz" (truncate) """ diff --git a/src/chop/nn/quantizers/_minifloat_mx/minifloat.py b/src/chop/nn/quantizers/_minifloat_mx/minifloat.py index c722ffc79..dfb4a97d4 100644 --- a/src/chop/nn/quantizers/_minifloat_mx/minifloat.py +++ b/src/chop/nn/quantizers/_minifloat_mx/minifloat.py @@ -6,7 +6,7 @@ from torch import Tensor from .meta import MinifloatMeta, MinifloatTensorMeta -from .fake import extract_minifloat_component, compose_minifloat_component +from .fake import quantize_minifloat_value def minifloat_quantizer_sim( @@ -25,9 +25,5 @@ def minifloat_quantizer_sim( Returns: Dequantized tensor """ - ori_dtype = tensor.dtype - element = extract_minifloat_component(tensor, minifloat_meta) - - return compose_minifloat_component( - element, minifloat_meta, output_dtype=output_dtype or ori_dtype - ) + value = quantize_minifloat_value(tensor, minifloat_meta) + return value.to(output_dtype or tensor.dtype) diff --git a/src/chop/nn/quantizers/mxfp/fake.py b/src/chop/nn/quantizers/mxfp/fake.py index a4aca7c1f..7c3128698 100644 --- a/src/chop/nn/quantizers/mxfp/fake.py +++ b/src/chop/nn/quantizers/mxfp/fake.py @@ -1,49 +1,59 @@ -""" -Fake MXFP quantization — aligned with Plena-Acc-Sim. - -""" +"""PLENA MXFP quantize-dequantize helpers.""" import torch from torch import Tensor -from ..minifloat import minifloat_denorm_quantizer +from .._minifloat_mx import minifloat_quantizer_sim from .meta import MXFPMeta def extract_mxfp_components( x: Tensor, mxfp_meta: MXFPMeta, percentile: float = 1.0 ) -> tuple[Tensor, Tensor]: - """Extract MXFP components (Plena-aligned). + """Split a blocked tensor into shared scales and minifloat elements. Returns: - scales: per-block shared exponent in **log2 domain** (float), - shape ``[n_blocks, 1]``. Recompose with ``2 ** scales``. - elements: per-element minifloat-quantized values (float, same dtype - as ``x``), shape ``[n_blocks, B]``. Already in scaled - (block-relative) space — multiply by ``2 ** scales`` to - get the dequantized tensor. + scales: per-block shared exponent in log2 domain (float), shape + ``[n_blocks, 1]``. Recompose with ``2 ** scales``. + elements: per-element minifloat values in block-relative space, + shape ``[n_blocks, B]``. """ B = mxfp_meta.block_size - assert x.numel() % B == 0, ( - f"Input tensor size {x.numel()} is not divisible by block size {B}." - ) + assert ( + x.numel() % B == 0 + ), f"Input tensor size {x.numel()} is not divisible by block size {B}." n_blocks = x.numel() // B x = x.flatten().reshape(n_blocks, B) + magnitude = x.abs().to(torch.float32) + # quantile() sorts each block; at percentile 1.0 the result is the maximum, + # which is also the only path that avoids quantile's input-size limit. per_block_max = ( - x.abs().to(torch.float32).quantile(percentile, dim=1, keepdim=True) + 1e-9 + magnitude.amax(dim=1, keepdim=True) + if percentile == 1.0 + else magnitude.quantile(percentile, dim=1, keepdim=True) + ) + nonzero_block = per_block_max > 0 + safe_block_max = torch.where( + nonzero_block, + per_block_max, + torch.ones_like(per_block_max), ) - scales = per_block_max.log2().ceil() + # OCP MX shared exponent: place the block maximum at the top of the + # element format's exponent range, so the elements use their full range. + scales = safe_block_max.log2().floor() - mxfp_meta.element_max_exponent + scale_bias = 2 ** (mxfp_meta.scale_exp_bits - 1) - 1 scales = scales.clamp( - min=-(2 ** (mxfp_meta.scale_exp_bits - 1)), - max=2 ** (mxfp_meta.scale_exp_bits - 1) - 1, + min=-scale_bias, + max=2**mxfp_meta.scale_exp_bits - 1 - scale_bias, ) + scales = torch.where(nonzero_block, scales, torch.zeros_like(scales)) q_tensor = x / 2**scales - elements = minifloat_denorm_quantizer( + elements = minifloat_quantizer_sim( q_tensor, - width=mxfp_meta.element_frac_bits + mxfp_meta.element_exp_bits + 1, - exponent_width=mxfp_meta.element_exp_bits, + minifloat_meta=mxfp_meta.element_meta, + output_dtype=x.dtype, ) return scales, elements @@ -55,13 +65,11 @@ def compose_mxfp_tensor( mxfp_meta: MXFPMeta, output_dtype: torch.dtype, ) -> Tensor: - """Reconstruct the dequantized tensor from Plena-style MXFP components. + """Reconstruct the dequantized tensor from MXFP components. - ``elements`` are the already-block-relative minifloat values; the shared - log2 scale just shifts them back into the original block magnitude. - ``mxfp_meta`` is unused for the recompose itself (kept in the signature - so callers don't need to know whether the impl is Plena- or OCP-style). + ``elements`` are block-relative minifloat values; the shared log2 scale + shifts them back into the original block magnitude. """ - del mxfp_meta # only needed by the OCP-style impl + del mxfp_meta # shape and format already fixed by extraction dequantized = (elements * 2**shared_scales).flatten().to(output_dtype) return dequantized diff --git a/src/chop/nn/quantizers/mxfp/helpers.py b/src/chop/nn/quantizers/mxfp/helpers.py index 1c6deb197..a65568de6 100644 --- a/src/chop/nn/quantizers/mxfp/helpers.py +++ b/src/chop/nn/quantizers/mxfp/helpers.py @@ -1,63 +1,56 @@ -""" -Helper functions for MX-format quantizers. -""" +"""Shape helpers shared by MX-format quantizers.""" +import torch from torch import Tensor -def flatten_for_quantize(tensor: Tensor, block_dim: int) -> Tensor: - """ - Permute tensor to move block dimension to last position and flatten. - - Args: - tensor: Input tensor - block_dim: Dimension to use for blocking +def block_rows_for_quantize( + tensor: Tensor, + block_dim: int, + block_size: int, +) -> tuple[Tensor, int]: + """Return independently padded rows with the block axis last. - Returns: - Flattened tensor with block_dim moved to last position + Zero padding is local to each logical row and therefore cannot merge the + tail of one attention head with the start of another. """ + if tensor.ndim == 0: + raise ValueError("MX block quantization requires at least one dimension") + if block_size <= 0: + raise ValueError("MX block size must be positive") ori_shape = tuple(tensor.shape) ndim = len(ori_shape) block_dim = block_dim % ndim - # Create permutation to move block_dim to last position permute = list(range(ndim)) permute.append(permute.pop(block_dim)) - - tensor = tensor.permute(permute) - tensor = tensor.flatten() - return tensor - - -def permute_for_dequantize( - flatten_tensor: Tensor, + rows = tensor.permute(permute).contiguous() + axis_size = rows.shape[-1] + padded_axis_size = ((axis_size + block_size - 1) // block_size) * block_size + if padded_axis_size != axis_size: + padded_shape = (*rows.shape[:-1], padded_axis_size) + padded = torch.zeros(padded_shape, dtype=rows.dtype, device=rows.device) + padded[..., :axis_size] = rows + rows = padded + return rows.reshape(-1, block_size), padded_axis_size + + +def restore_quantized_rows( + block_tensor: Tensor, ori_shape: tuple[int, ...], block_dim: int, + padded_axis_size: int, ) -> Tensor: - """ - Reshape flattened tensor back to original shape after dequantization. - - Args: - flatten_tensor: Flattened tensor from quantization - ori_shape: Original tensor shape before flattening - block_dim: Original block dimension - - Returns: - Tensor restored to original shape - """ + """Remove row-local padding and restore the original dimension order.""" ndim = len(ori_shape) block_dim = block_dim % ndim - # Create the shape after moving block_dim to last position permuted_shape = list(ori_shape) permuted_shape.append(permuted_shape.pop(block_dim)) + axis_size = permuted_shape[-1] + padded_shape = (*permuted_shape[:-1], padded_axis_size) + tensor = block_tensor.reshape(padded_shape)[..., :axis_size] - # Reshape from flattened form to intermediate permuted form - tensor = flatten_tensor.reshape(permuted_shape) - - # Create inverse permutation to restore original dimension order inverse_permute = list(range(ndim)) inverse_permute.insert(block_dim, inverse_permute.pop(-1)) - - tensor = tensor.permute(inverse_permute) - return tensor + return tensor.permute(inverse_permute).contiguous() diff --git a/src/chop/nn/quantizers/mxfp/meta.py b/src/chop/nn/quantizers/mxfp/meta.py index fcd66bf26..dca7ed580 100644 --- a/src/chop/nn/quantizers/mxfp/meta.py +++ b/src/chop/nn/quantizers/mxfp/meta.py @@ -21,7 +21,7 @@ class MXFPMeta: scale_exp_bits: Bits for shared scale exponent (typically 8) element_exp_bits: Exponent bits per element (e.g., 4 for E4M3) element_frac_bits: Fraction bits per element (e.g., 3 for E4M3) - element_is_finite: Whether elements support inf/nan + element_is_finite: Whether every element exponent code is finite round_mode: Rounding mode """ @@ -38,7 +38,17 @@ def __post_init__(self): f"Invalid scale exponent bits: {self.scale_exp_bits}. " f"Legal values are: {legal_scale_exp_bits}." ) - legal_element_exp_frac_bits = ((4, 3), (5, 2), (2, 3), (3, 2), (2, 1), (1, 2)) + # (exp_bits, frac_bits) element formats in the accelerator search + # space: 4-bit E1M2/E2M1, 6-bit E2M3/E3M2, 8-bit E3M4/E4M3/E5M2. + legal_element_exp_frac_bits = ( + (1, 2), + (2, 1), + (2, 3), + (3, 2), + (3, 4), + (4, 3), + (5, 2), + ) el_exp_frac = (self.element_exp_bits, self.element_frac_bits) assert el_exp_frac in legal_element_exp_frac_bits, ( f"Invalid element exp/frac bits: {el_exp_frac}. " @@ -55,6 +65,14 @@ def element_meta(self) -> MinifloatMeta: round_mode=self.round_mode, ) + @functools.cached_property + def element_max_exponent(self) -> int: + """Largest unbiased exponent the element format can represent.""" + exp_bits = self.element_exp_bits + bias = 1 if exp_bits == 1 else (1 << (exp_bits - 1)) - 1 + max_code = (1 << exp_bits) - (1 if self.element_is_finite else 2) + return max_code - bias + @dataclass(frozen=True) class MXFPTensorMeta: diff --git a/src/chop/nn/quantizers/mxfp/mxfp.py b/src/chop/nn/quantizers/mxfp/mxfp.py index 12f9da3bd..ff7d3981a 100644 --- a/src/chop/nn/quantizers/mxfp/mxfp.py +++ b/src/chop/nn/quantizers/mxfp/mxfp.py @@ -7,7 +7,7 @@ from tqdm import tqdm from .meta import MXFPMeta, MXFPTensorMeta -from .helpers import flatten_for_quantize, permute_for_dequantize +from .helpers import block_rows_for_quantize, restore_quantized_rows from .fake import extract_mxfp_components, compose_mxfp_tensor @@ -35,27 +35,24 @@ def mxfp_quantizer_sim( Returns: Dequantized tensor """ - out_dq = torch.zeros_like(tensor) - if quantile_search: - qtensor = tensor.flatten() B = mxfp_meta.block_size - - qtensor = qtensor.reshape(-1, B) + qtensor, padded_axis_size = block_rows_for_quantize(tensor, block_dim, B) best = torch.full( [qtensor.shape[0]], float("inf"), device=tensor.device, dtype=tensor.dtype ) - best_scales, best_elements, tensor_meta = _extract_with_meta( + best_scales, best_elements, tensor_meta, padded_axis_size = _extract_with_meta( tensor, block_dim, mxfp_meta, percentile=1.0 ) percentiles = [1.0, 0.995, 0.99, 0.97, 0.95, 0.93, 0.90, 0.80, 0.70, 0.60, 0.50] for percentile in percentiles: - scales, elements, tensor_meta = _extract_with_meta( - tensor, block_dim, mxfp_meta, percentile=percentile + scales, elements, tensor_meta, candidate_padded_axis_size = ( + _extract_with_meta(tensor, block_dim, mxfp_meta, percentile=percentile) ) - # Plena: elements are already block-relative minifloat values, - # scales are raw log2-domain exponents — dequant is just scale-back. + if candidate_padded_axis_size != padded_axis_size: + raise RuntimeError("MXFP row padding changed during quantile search") + # Elements are block-relative; dequant is a plain scale-back. q = (elements * 2**scales).to(dtype=qtensor.dtype) if act_tensor is not None: @@ -100,14 +97,19 @@ def mxfp_quantizer_sim( best_elements[tmp] = elements[tmp] else: - best_scales, best_elements, tensor_meta = _extract_with_meta( + best_scales, best_elements, tensor_meta, padded_axis_size = _extract_with_meta( tensor, block_dim, mxfp_meta, percentile=1.0 ) out_dq = compose_mxfp_tensor( best_scales, best_elements, tensor_meta.meta, output_dtype=dtype or tensor.dtype ) - out_dq = permute_for_dequantize(out_dq, tensor_meta.shape, tensor_meta.block_dim) + out_dq = restore_quantized_rows( + out_dq, + tensor_meta.shape, + tensor_meta.block_dim, + padded_axis_size, + ) return out_dq @@ -116,7 +118,7 @@ def _extract_with_meta( block_dim: int, mxfp_meta: MXFPMeta, percentile: float = 1.0, -) -> tuple[Tensor, Tensor, MXFPTensorMeta]: +) -> tuple[Tensor, Tensor, MXFPTensorMeta, int]: """Extract MXFP components with tensor metadata.""" device = str(tensor.device) ori_shape = tuple(tensor.shape) @@ -124,9 +126,11 @@ def _extract_with_meta( ndim = len(ori_shape) assert block_dim < ndim and block_dim >= -ndim - tensor_flat = flatten_for_quantize(tensor, block_dim) + tensor_blocks, padded_axis_size = block_rows_for_quantize( + tensor, block_dim, mxfp_meta.block_size + ) scales, elements = extract_mxfp_components( - tensor_flat, mxfp_meta, percentile=percentile + tensor_blocks, mxfp_meta, percentile=percentile ) tensor_meta = MXFPTensorMeta( @@ -136,7 +140,7 @@ def _extract_with_meta( block_dim=block_dim, meta=mxfp_meta, ) - return scales, elements, tensor_meta + return scales, elements, tensor_meta, padded_axis_size # ============================================================================= @@ -163,7 +167,7 @@ def forward( scale_exp_bits=scale_exp_bits, element_exp_bits=element_exp_bits, element_frac_bits=element_frac_bits, - element_is_finite=True, + element_is_finite=(element_exp_bits == 1), round_mode="rn", ) return mxfp_quantizer_sim( diff --git a/src/chop/nn/quantizers/mxint/fake.py b/src/chop/nn/quantizers/mxint/fake.py index c5b1b2a41..bc5c9800c 100644 --- a/src/chop/nn/quantizers/mxint/fake.py +++ b/src/chop/nn/quantizers/mxint/fake.py @@ -8,6 +8,28 @@ from .meta import MXIntMeta +def mxint_shared_exponent(maximum: Tensor, element_bits: int) -> Tensor: + """Return the smallest exponent whose sign-magnitude range covers the block.""" + magnitude_levels = 2 ** (element_bits - 1) + qmax = (magnitude_levels - 1) / magnitude_levels + return (maximum.to(torch.float32) / qmax).log2().ceil() + + +def mxint_sign_magnitude_codes(elements: Tensor, element_bits: int) -> Tensor: + """Encode signed integer elements as canonical MXINT sign-magnitude codes.""" + if element_bits not in (2, 4, 8): + raise ValueError("MXINT element width must be 2, 4, or 8") + integral = elements.round() + if not torch.equal(elements, integral): + raise ValueError("MXINT elements must be integral") + magnitude = integral.abs().to(torch.int64) + magnitude_max = 2 ** (element_bits - 1) - 1 + if torch.any(magnitude > magnitude_max): + raise ValueError("MXINT magnitude is outside its declared width") + sign = ((integral < 0) & (magnitude != 0)).to(torch.int64) + return (sign << (element_bits - 1)) | magnitude + + def extract_mxint_components( x: Tensor, mxint_meta: MXIntMeta, percentile: float = 1.0 ) -> tuple[Tensor, Tensor]: @@ -32,26 +54,33 @@ def extract_mxint_components( x = x.reshape(n_blocks, B) ori_dtype = x.dtype - # quantile needs fp32 + # quantile needs fp32; at percentile 1.0 it reduces to the block maximum, + # which avoids sorting every block and quantile's input-size limit. + magnitude = x.abs().to(torch.float32) x_max = ( - x.abs() - .to(torch.float32) - .quantile(percentile, dim=1, keepdim=True) - .to(ori_dtype) - ) + magnitude.amax(dim=1, keepdim=True) + if percentile == 1.0 + else magnitude.quantile(percentile, dim=1, keepdim=True) + ).to(ori_dtype) - # Clamp to avoid log2(0) = -inf for all-zero blocks - x_max = x_max.clamp(min=torch.finfo(x_max.dtype).tiny) - scale = x_max.log2().ceil() + zero_blocks = x_max == 0 scale_bias = 2 ** (mxint_meta.scale_bits - 1) - 1 + scale_min = -scale_bias + scale_max = 2**mxint_meta.scale_bits - 1 - scale_bias + unit_max = torch.where(zero_blocks, torch.ones_like(x_max), x_max) + scale = mxint_shared_exponent( + unit_max, + mxint_meta.element_bits, + ).clamp(min=scale_min, max=scale_max) + scale = torch.where(zero_blocks, torch.zeros_like(scale), scale) x = x / 2**scale x_mant = x * 2 ** (mxint_meta.element_bits - 1) - scale = scale + scale_bias - scale = scale.clamp(min=0, max=2**mxint_meta.scale_bits - 1) + magnitude_max = 2 ** (mxint_meta.element_bits - 1) - 1 x_mant = x_mant.round().clamp( - min=-(2 ** (mxint_meta.element_bits - 1)), - max=2 ** (mxint_meta.element_bits - 1) - 1, + min=-magnitude_max, + max=magnitude_max, ) + scale = scale + scale_bias return scale, x_mant diff --git a/src/chop/nn/quantizers/mxint/mxint.py b/src/chop/nn/quantizers/mxint/mxint.py index 4f08d4e04..cf0c841c6 100644 --- a/src/chop/nn/quantizers/mxint/mxint.py +++ b/src/chop/nn/quantizers/mxint/mxint.py @@ -7,8 +7,12 @@ from tqdm import tqdm from .meta import MXIntMeta, MXIntTensorMeta -from .fake import extract_mxint_components, compose_mxint_tensor -from ..mxfp.helpers import flatten_for_quantize, permute_for_dequantize +from .fake import ( + compose_mxint_tensor, + extract_mxint_components, + mxint_shared_exponent, +) +from ..mxfp.helpers import block_rows_for_quantize, restore_quantized_rows def mxint_quantizer_sim( @@ -38,10 +42,8 @@ def mxint_quantizer_sim( tensor_dtype = tensor.dtype if quantile_search: - qtensor = tensor.flatten() B = mxint_meta.block_size - - qtensor = qtensor.reshape(-1, B) + qtensor, padded_axis_size = block_rows_for_quantize(tensor, block_dim, B) percentiles = torch.tensor( [1.0, 0.995, 0.99, 0.97, 0.95, 0.93, 0.90, 0.80, 0.70, 0.60, 0.50], @@ -55,13 +57,8 @@ def mxint_quantizer_sim( ndim = len(ori_shape) assert block_dim < ndim and block_dim >= -ndim - tensor_flat = flatten_for_quantize(tensor, block_dim) - - x = tensor_flat - n_blocks = x.numel() // B - - x = x.flatten() - x = x.reshape(n_blocks, B) + x = qtensor + n_blocks = x.shape[0] tem_dtype = x.dtype x_max = ( @@ -71,18 +68,23 @@ def mxint_quantizer_sim( .to(tem_dtype) ) - # Clamp to avoid log2(0) = -inf for all-zero blocks - x_max = x_max.clamp(min=torch.finfo(x_max.dtype).tiny) - scale = x_max.log2().ceil() + zero_blocks = x_max == 0 scale_bias = 2 ** (mxint_meta.scale_bits - 1) - 1 + scale_min = -scale_bias + scale_max = 2**mxint_meta.scale_bits - 1 - scale_bias + unit_max = torch.where(zero_blocks, torch.ones_like(x_max), x_max) + scale = mxint_shared_exponent( + unit_max, + mxint_meta.element_bits, + ).clamp(min=scale_min, max=scale_max) x = x / 2**scale x_mant = x * 2 ** (mxint_meta.element_bits - 1) - scale = scale + scale_bias - scale = scale.clamp(min=0, max=2**mxint_meta.scale_bits - 1) + magnitude_max = 2 ** (mxint_meta.element_bits - 1) - 1 x_mant = x_mant.round().clamp( - min=-(2 ** (mxint_meta.element_bits - 1)), - max=2 ** (mxint_meta.element_bits - 1) - 1, + min=-magnitude_max, + max=magnitude_max, ) + scale = scale + scale_bias quant_tensor = ( x_mant / 2 ** (mxint_meta.element_bits - 1) * 2 ** (scale - scale_bias) @@ -137,8 +139,11 @@ def mxint_quantizer_sim( meta=mxint_meta, ) - tensor_out = permute_for_dequantize( - quant_tensor, ori_shape=tensor_meta.shape, block_dim=tensor_meta.block_dim + tensor_out = restore_quantized_rows( + quant_tensor, + ori_shape=tensor_meta.shape, + block_dim=tensor_meta.block_dim, + padded_axis_size=padded_axis_size, ) out_dq = tensor_out.to(tensor_dtype) @@ -149,9 +154,11 @@ def mxint_quantizer_sim( ndim = len(ori_shape) assert block_dim < ndim and block_dim >= -ndim - tensor_flat = flatten_for_quantize(tensor, block_dim) + tensor_blocks, padded_axis_size = block_rows_for_quantize( + tensor, block_dim, mxint_meta.block_size + ) scales, elements = extract_mxint_components( - tensor_flat, mxint_meta, percentile=1.0 + tensor_blocks, mxint_meta, percentile=1.0 ) tensor_meta = MXIntTensorMeta( @@ -163,8 +170,11 @@ def mxint_quantizer_sim( ) dequant = compose_mxint_tensor(scales, elements, mxint_meta) - out_dq = permute_for_dequantize( - dequant, tensor_meta.shape, tensor_meta.block_dim + out_dq = restore_quantized_rows( + dequant, + tensor_meta.shape, + tensor_meta.block_dim, + padded_axis_size, ) out_dq = out_dq.to(tensor_dtype) @@ -224,8 +234,8 @@ def mxint_quantizer( Handles DTensor inputs transparently (unwraps, quantizes local shard, re-wraps with the same placement). MX is a pointwise-on-blocks operation, - so each rank can quantize its own shard independently as long as - shard_size is divisible by block_size along ``block_dim``. + so each rank quantizes its own shard with row-local tail padding along + ``block_dim``. Args: x: Input tensor to quantize (torch.Tensor or DTensor) @@ -259,24 +269,20 @@ def mxint_quantizer( mesh = x.device_mesh local = x.to_local() local_q = mxint_quantizer( - local, block_size, element_bits, block_dim, scale_bits, quantile_search, + local, + block_size, + element_bits, + block_dim, + scale_bits, + quantile_search, ) return DTensor.from_local(local_q, mesh, placements) - # Handle tensors whose total element count isn't a multiple of block_size - # (common in MoE expert forwards where n_matched_tokens is arbitrary). - # Pad with zeros at the end, quantize as 1D, then trim + reshape back. - if x.numel() % block_size != 0: - import torch.nn.functional as F - orig_shape = x.shape - orig_numel = x.numel() - pad = block_size - (orig_numel % block_size) - x_padded = F.pad(x.reshape(-1), (0, pad)) - q_padded = MXIntQuantize.apply( - x_padded, block_size, element_bits, -1, scale_bits, quantile_search, - ) - return q_padded[:orig_numel].reshape(orig_shape) - return MXIntQuantize.apply( - x, block_size, element_bits, block_dim, scale_bits, quantile_search, + x, + block_size, + element_bits, + block_dim, + scale_bits, + quantile_search, ) diff --git a/test/nn/quantizers/test_mx_block_quantization.py b/test/nn/quantizers/test_mx_block_quantization.py new file mode 100644 index 000000000..4962b2bb7 --- /dev/null +++ b/test/nn/quantizers/test_mx_block_quantization.py @@ -0,0 +1,84 @@ +"""Regression tests for MX block quantization correctness.""" + +import torch + +from chop.nn.quantizers.mxfp.mxfp import mxfp_quantizer +from chop.nn.quantizers.mxint.mxint import mxint_quantizer + +MXFP_FORMATS = [(1, 2), (2, 1), (2, 3), (3, 2), (3, 4), (4, 3), (5, 2)] +MXINT_WIDTHS = [2, 4, 8] + + +def test_blocks_do_not_straddle_rows(): + """A block axis that is not a multiple of the block size must still + keep each row independent: perturbing one row cannot change another.""" + torch.manual_seed(0) + block_size, rows, axis = 32, 4, 80 # 80 % 32 != 0 + x = torch.randn(rows, axis) + kwargs = dict( + block_size=block_size, + element_exp_bits=4, + element_frac_bits=3, + block_dim=-1, + ) + baseline = mxfp_quantizer(x, **kwargs) + perturbed = x.clone() + perturbed[0] *= 100.0 + result = mxfp_quantizer(perturbed, **kwargs) + for row in range(1, rows): + assert torch.equal( + baseline[row], result[row] + ), f"row {row} changed when only row 0 was perturbed" + + +def test_mxfp_requantization_is_idempotent(): + """Quantizing an already-quantized tensor must be a no-op, because a + cached value is re-read many times after one quantization round trip.""" + for exp_bits, frac_bits in MXFP_FORMATS: + torch.manual_seed(0) + x = torch.randn(256, 512) + kwargs = dict( + block_size=32, + element_exp_bits=exp_bits, + element_frac_bits=frac_bits, + block_dim=-1, + ) + once = mxfp_quantizer(x, **kwargs) + twice = mxfp_quantizer(once, **kwargs) + assert torch.equal(once, twice), f"E{exp_bits}M{frac_bits} not idempotent" + + +def test_mxint_requantization_is_idempotent(): + for element_bits in MXINT_WIDTHS: + torch.manual_seed(0) + x = torch.randn(256, 512) + kwargs = dict(block_size=32, element_bits=element_bits, block_dim=-1) + once = mxint_quantizer(x, **kwargs) + twice = mxint_quantizer(once, **kwargs) + assert torch.equal(once, twice), f"MXINT{element_bits} not idempotent" + + +def test_shape_and_dtype_are_preserved(): + torch.manual_seed(0) + for shape, block_dim in [((8, 256), -1), ((80, 4), 0), ((4, 80), -1)]: + x = torch.randn(*shape) + out = mxfp_quantizer( + x, + block_size=32, + element_exp_bits=4, + element_frac_bits=3, + block_dim=block_dim, + ) + assert out.shape == x.shape + assert out.dtype == x.dtype + + +def test_small_block_maxima_are_not_flushed_to_zero(): + """A block whose maximum is small but nonzero must not be zeroed, + including at half precision.""" + for dtype in (torch.float16, torch.bfloat16, torch.float32): + torch.manual_seed(0) + x = (torch.randn(4, 64) * 1e-7).to(dtype) + assert x.abs().amax() > 0 + out = mxint_quantizer(x, block_size=32, element_bits=4, block_dim=-1) + assert not bool((out == 0).all()), f"{dtype} block flushed to zero"