Repository navigation
Expand file tree
/
Copy pathengine.py
More file actions
140 lines (116 loc) · 5.18 KB
/
Copy pathengine.py
File metadata and controls
140 lines (116 loc) · 5.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
from typing import Iterable
import math
import torch
from tqdm import tqdm
def _reset_bn_stats(model: torch.nn.Module):
for module in model.modules():
if isinstance(module, (torch.nn.BatchNorm1d, torch.nn.BatchNorm2d, torch.nn.BatchNorm3d)):
if torch.isnan(module.running_mean).any() or torch.isnan(module.running_var).any():
module.running_mean.zero_()
module.running_var.fill_(1.0)
def train_one_epoch(
model: torch.nn.Module,
criterion: torch.nn.Module,
data_loader: Iterable,
optimizer: torch.optim.Optimizer,
epoch: int,
args=None,
logger=None,
log_interval: int = 50,
save_vis_interval: int = 9999
):
model.train()
criterion.train()
model.to(args.device)
criterion.to(args.device)
if hasattr(model, 'set_train_mode'):
model.set_train_mode(True)
modal_num = len(args.modals)
warmup_epochs = getattr(args, 'dualflow_warmup_epochs', 0)
use_dual_flow = (epoch >= warmup_epochs)
max_grad_norm = getattr(args, 'max_grad_norm', 1.0)
pbar = tqdm(total=len(data_loader.dataloader), desc=f"Epoch {epoch}")
epoch_losses = {'total': 0.0, 'seg_f': 0.0}
if use_dual_flow:
epoch_losses.update({'seg_p': 0.0, 'imd': 0.0, 'mgc': 0.0, 'fcl': 0.0})
num_batches = 0
for j, data in enumerate(data_loader):
modal_imgs = []
for i in range(modal_num):
items = list(data.items())
key, value = items[i]
modal_imgs.append(value.to(args.device))
targets = data['label'].to(args.device)
modal_mask = data['modal_mask'].to(args.device) if 'modal_mask' in data else None
if use_dual_flow:
P_f, distill_outputs = model(
modal_imgs, gt_mask=targets, return_distill_outputs=True,
modal_mask=modal_mask
)
if torch.isnan(P_f).any():
if logger:
logger.warning(f"Epoch {epoch} Batch {j}: NaN in model output, resetting BN stats and skipping")
_reset_bn_stats(model)
pbar.update(1)
continue
effective_distill = distill_outputs if (distill_outputs and 'P_p' in distill_outputs) else None
losses = criterion(P_f, targets.float(), effective_distill)
loss_final = losses['total']
for key in ['seg_f', 'seg_p', 'imd', 'mgc', 'fcl']:
if key in losses:
val = losses[key]
epoch_losses[key] = epoch_losses.get(key, 0.0) + (
val.item() if isinstance(val, torch.Tensor) else val
)
else:
output_fused, extras = model(modal_imgs, gt_mask=targets, modal_mask=modal_mask)
if torch.isnan(output_fused).any():
if logger:
logger.warning(f"Epoch {epoch} Batch {j}: NaN in model output, resetting BN stats and skipping")
_reset_bn_stats(model)
pbar.update(1)
continue
losses = criterion(output_fused, targets.float())
loss_final = losses['total']
epoch_losses['seg_f'] += losses.get('seg_f', torch.tensor(0.0)).item()
epoch_losses['total'] += loss_final.item()
num_batches += 1
if j % log_interval == 0:
lr = optimizer.param_groups[0]['lr']
if use_dual_flow:
logger.info(
f"Epoch {epoch} | Batch {j}/{len(data_loader.dataloader)} | "
f"Loss: {loss_final.item():.4f} | Seg_f: {losses.get('seg_f', 0):.4f} | "
f"Seg_p: {losses.get('seg_p', 0):.4f} | "
f"IMD: {losses.get('imd', 0):.4f} | MGC: {losses.get('mgc', 0):.4f} | "
f"FCL: {losses.get('fcl', 0):.4f} | LR: {lr:.6f}"
)
else:
logger.info(
f"Epoch {epoch} [Warmup] | Batch {j}/{len(data_loader.dataloader)} | "
f"Loss: {loss_final.item():.4f} | LR: {lr:.6f}"
)
optimizer.zero_grad()
if torch.isnan(loss_final) or torch.isinf(loss_final):
logger.warning(f"Epoch {epoch} Batch {j}: NaN/Inf loss detected, skipping batch")
continue
loss_final.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
optimizer.step()
pbar.set_postfix({
})
pbar.update(1)
pbar.close()
if num_batches > 0:
avg_losses = {k: v / num_batches for k, v in epoch_losses.items()}
logger.info("=" * 60)
phase_str = "Dual-Flow" if use_dual_flow else "Warmup"
logger.info(f"Epoch {epoch} Summary ({phase_str}):")
logger.info(f" Avg Total Loss: {avg_losses['total']:.4f}")
logger.info(f" Avg Seg_f Loss: {avg_losses['seg_f']:.4f}")
if use_dual_flow:
logger.info(f" Avg Seg_p Loss: {avg_losses.get('seg_p', 0):.4f}")
logger.info(f" Avg IMD Loss: {avg_losses.get('imd', 0):.4f}")
logger.info(f" Avg MGC Loss: {avg_losses.get('mgc', 0):.4f}")
logger.info(f" Avg FCL Loss: {avg_losses.get('fcl', 0):.4f}")
logger.info("=" * 60)