Repository navigation
Expand file tree
/
Copy pathpruning.py
More file actions
65 lines (53 loc) · 2.47 KB
/
Copy pathpruning.py
File metadata and controls
65 lines (53 loc) · 2.47 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
"""Weight pruning: zero out the smallest-magnitude weights (global, unstructured),
then fine-tune with the mask held fixed to recover accuracy."""
import copy
import torch
from torch import nn
from common.utils import (stage, MODEL_DIR, accuracy, gzip_size_bytes, load_data, log, make_mlp,
save_metrics, train_model)
from techniques.baseline import TEACHER_HIDDEN
SPARSITIES = [0.0, 0.5, 0.8, 0.9, 0.95, 0.98]
def global_magnitude_masks(model, sparsity):
weights = {m.weight: m.weight for m in model if isinstance(m, nn.Linear)}
if sparsity == 0:
return {w: torch.ones_like(w) for w in weights}
all_abs = torch.cat([w.detach().abs().flatten() for w in weights])
k = int(sparsity * all_abs.numel())
threshold = torch.kthvalue(all_abs, k).values
return {w: (w.detach().abs() > threshold).float() for w in weights}
def sparsity_of(model):
ws = [m.weight for m in model if isinstance(m, nn.Linear)]
zeros = sum(int((w == 0).sum()) for w in ws)
return zeros / sum(w.numel() for w in ws)
@stage(4, "Weight pruning")
def main():
Xtr, ytr, Xte, yte = load_data()
teacher = make_mlp(Xtr.shape[1], TEACHER_HIDDEN, int(ytr.max()) + 1)
teacher.load_state_dict(torch.load(MODEL_DIR / "teacher.pt"))
rows = []
for s in SPARSITIES:
log.info("--- Pruning to %.0f%% sparsity (global magnitude threshold) ---", s * 100)
model = copy.deepcopy(teacher)
masks = global_magnitude_masks(model, s)
with torch.no_grad():
for w, m in masks.items():
w.mul_(m)
acc_pruned = accuracy(model, Xte, yte)
log.info("Zeroed weights per layer: %s | accuracy right after pruning=%.4f",
[f"{float((m.weight == 0).float().mean()):.0%}" for m in model
if isinstance(m, nn.Linear)], acc_pruned)
if s > 0:
log.info("Fine-tuning with mask fixed to recover accuracy")
train_model(model, Xtr, ytr, epochs=5, lr=3e-4, masks=masks)
rows.append({
"target_sparsity": s,
"actual_sparsity": round(sparsity_of(model), 4),
"acc_after_prune": round(acc_pruned, 4),
"acc_after_finetune": round(accuracy(model, Xte, yte), 4),
"gzip_size_kb": round(gzip_size_bytes(model.state_dict()) / 1024, 1),
})
if s == 0.9:
torch.save(model.state_dict(), MODEL_DIR / "pruned_90.pt")
save_metrics("pruning", rows)
if __name__ == "__main__":
main()