Repository navigation
Expand file tree
/
Copy pathquantization.py
More file actions
89 lines (70 loc) · 3.89 KB
/
Copy pathquantization.py
File metadata and controls
89 lines (70 loc) · 3.89 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
"""Dynamic range quantization, implemented by hand so each step is visible.
- Weights: quantized ahead of time to int8 (symmetric, per-output-channel scale).
- Activations: quantized on the fly at inference using the min/max range of each batch
("dynamic range"), so no calibration dataset or retraining is needed.
Integer matmuls are simulated in float32 (int8 values are exactly representable), which keeps the
lab portable across CPUs/architectures. Size and accuracy are real; latency is intentionally not
reported because real speedups need an int8 kernel (e.g. fbgemm, XNNPACK, TFLite)."""
import torch
from torch import nn
from common.utils import (stage, MODEL_DIR, accuracy, load_data, log, make_mlp, param_count,
save_metrics, size_bytes)
from techniques.baseline import TEACHER_HIDDEN
def quantize_weight(w):
scale = (w.abs().amax(dim=1, keepdim=True) / 127).clamp_min(1e-12)
return torch.round(w / scale).clamp(-127, 127).to(torch.int8), scale.squeeze(1)
def quantize_activation(x):
"""Asymmetric uint8 quantization using the observed range of this batch."""
lo, hi = x.min().clamp(max=0), x.max().clamp(min=0)
scale = ((hi - lo) / 255).clamp_min(1e-12)
zero_point = torch.round(-lo / scale)
q = torch.round(x / scale + zero_point).clamp(0, 255)
return q, scale, zero_point
class DynamicQuantLinear(nn.Module):
def __init__(self, linear):
super().__init__()
self.w_int8, self.w_scale = quantize_weight(linear.weight.detach())
self.bias = linear.bias.detach().clone()
def forward(self, x):
q, a_scale, zp = quantize_activation(x)
acc = (q - zp) @ self.w_int8.float().T # int32 accumulation in a real kernel
return acc * (a_scale * self.w_scale) + self.bias
def state(self):
return {"w_int8": self.w_int8, "w_scale": self.w_scale, "bias": self.bias}
def quantize_dynamic(model):
return nn.Sequential(*[DynamicQuantLinear(m) if isinstance(m, nn.Linear) else m for m in model])
@stage(3, "Dynamic range quantization")
def main():
_, ytr, Xte, yte = load_data()
fp32 = make_mlp(Xte.shape[1], TEACHER_HIDDEN, int(ytr.max()) + 1)
fp32.load_state_dict(torch.load(MODEL_DIR / "teacher.pt"))
fp32.eval()
log.info("Quantizing %d Linear layers: weights fp32 -> int8 (per-channel scale); "
"activations quantized per batch at inference", sum(isinstance(m, nn.Linear) for m in fp32))
for i, m in enumerate(fp32):
if isinstance(m, nn.Linear):
q, s = quantize_weight(m.weight.detach())
err = (q.float() * s[:, None] - m.weight.detach()).abs()
log.info(" layer %d: weights in [%.3f, %.3f] -> int8 [%d, %d]; mean abs rounding "
"error=%.6f", i, m.weight.min(), m.weight.max(), q.min(), q.max(), err.mean())
sample = torch.from_numpy(Xte[:256])
_, a_scale, zp = quantize_activation(sample)
log.info("Example dynamic activation range (first batch input): scale=%.5f zero_point=%d",
a_scale, zp)
int8 = quantize_dynamic(fp32).eval()
agree = float((int8(sample).argmax(1) == fp32(sample).argmax(1)).float().mean())
log.info("int8 vs fp32 prediction agreement on 256 rows: %.1f%%", agree * 100)
int8_state = {f"layer{i}": m.state() for i, m in enumerate(int8) if isinstance(m, DynamicQuantLinear)}
torch.save(int8_state, MODEL_DIR / "quantized_int8.pt")
rows = [
{"model": "fp32", "params": param_count(fp32),
"size_kb": round(size_bytes(fp32.state_dict()) / 1024, 1),
"test_acc": round(accuracy(fp32, Xte, yte), 4)},
{"model": "dynamic_int8", "params": param_count(fp32),
"size_kb": round(size_bytes(int8_state) / 1024, 1),
"test_acc": round(accuracy(int8, Xte, yte), 4)},
]
rows[1]["size_reduction_x"] = round(rows[0]["size_kb"] / rows[1]["size_kb"], 2)
save_metrics("quantization", rows)
if __name__ == "__main__":
main()