Repository navigation
Expand file tree
/
Copy pathinitialize.py
More file actions
219 lines (174 loc) · 8.05 KB
/
Copy pathinitialize.py
File metadata and controls
219 lines (174 loc) · 8.05 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
import os
import sys
import wandb
import argparse
from omegaconf import OmegaConf
import torch
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from terediff.model import ControlLDM, SwinIR, Diffusion
from terediff.utils.common import instantiate_from_config, to, log_txt_as_img
from terediff.dataset.codeformer import collate_fn_code
from terediff.dataset.realesrgan import collate_fn_real
def load_experiment_settings(accelerator, cfg):
if cfg.exp_args.mode == 'TRAIN':
datasets='_'.join((cfg.dataset.train.params.data_args['datasets']))
exp_name = f"{cfg.exp_args.mode}_{cfg.exp_args.model_name}_{cfg.exp_args.finetuning_method}_bs{cfg.train.batch_size}_lr{cfg.train.learning_rate}"
# setup an experiment folder
exp_dir = cfg.train.exp_dir
os.makedirs(exp_dir, exist_ok=True)
ckpt_dir = os.path.join(exp_dir, exp_name)
os.makedirs(ckpt_dir, exist_ok=True)
elif cfg.exp_args.mode == 'VAL':
datasets = cfg.dataset.val_dataset_name
exp_name = f"{cfg.exp_args.mode}_{cfg.exp_args.model_name}"
exp_dir=None
ckpt_dir=None
if accelerator.is_main_process:
# setup logging tool
if cfg.log_args.log_tool == 'wandb':
wandb.login(key=cfg.log_args.wandb_key)
wandb.init(project=cfg.log_args.wandb_proj_name,
name=exp_name,
config=argparse.Namespace(**OmegaConf.to_container(cfg, resolve=True))
)
return exp_dir, ckpt_dir, exp_name, None
def load_data(accelerator, cfg):
# set dataset
train_ds = instantiate_from_config(cfg.dataset.train)
val_ds = instantiate_from_config(cfg.dataset.val)
if cfg.dataset.dataset_type == 'realsr':
collate_fn = collate_fn_real
# set data loader
train_loader = DataLoader(
dataset=train_ds,
batch_size=cfg.train.batch_size,
num_workers=cfg.train.num_workers,
shuffle=True,
drop_last=True,
pin_memory=True,
collate_fn=collate_fn
)
val_loader = DataLoader(
dataset=val_ds,
batch_size=cfg.val.batch_size,
num_workers=cfg.val.num_workers,
shuffle=False,
drop_last=True,
pin_memory=True,
collate_fn=collate_fn
)
return train_ds, val_ds, train_loader, val_loader
def load_model(accelerator, device, args, cfg):
loaded_models={}
# default: load cldm, swinir
cldm: ControlLDM = instantiate_from_config(cfg.model.cldm)
sd = torch.load(cfg.train.sd_path, map_location="cpu")["state_dict"]
unused, missing = cldm.load_pretrained_sd(sd)
if accelerator.is_main_process:
print(
f"strictly load pretrained SD weight from {cfg.train.sd_path}\n"
f"unused weights: {unused}\n"
f"missing weights: {missing}"
)
if cfg.train.resume:
cldm.load_controlnet_from_ckpt(torch.load(cfg.train.resume, map_location="cpu"))
if accelerator.is_main_process:
print(
f"strictly load controlnet weight from checkpoint: {cfg.train.resume}"
)
else:
init_with_new_zero, init_with_scratch = cldm.load_controlnet_from_unet()
if accelerator.is_main_process:
print(
f"strictly load controlnet weight from pretrained SD\n"
f"weights initialized with newly added zeros: {init_with_new_zero}\n"
f"weights initialized from scratch: {init_with_scratch}"
)
swinir: SwinIR = instantiate_from_config(cfg.model.swinir)
sd = torch.load(cfg.train.swinir_path, map_location="cpu")
if "state_dict" in sd:
sd = sd["state_dict"]
sd = {
(k[len("module.") :] if k.startswith("module.") else k): v
for k, v in sd.items()
}
swinir.load_state_dict(sd, strict=True)
for p in swinir.parameters():
p.requires_grad = False
if accelerator.is_main_process:
print(f"load SwinIR from {cfg.train.swinir_path}")
# set mode and cuda
loaded_models['cldm'] = cldm.train().to(device)
loaded_models['swinir'] = swinir.eval().to(device)
# load text spotting module
if cfg.exp_args.model_name == 'terediff_stage2' or cfg.exp_args.model_name == 'terediff_stage3':
sys.path.append(f'{os.getcwd()}/testr')
from testr.adet.modeling.transformer_detector import TransformerDetector
from testr.adet.config import get_cfg
# get testr config
config_testr = get_cfg()
config_testr.merge_from_file(args.config_testr)
config_testr.freeze()
# load testr model
detector = TransformerDetector(config_testr)
# load testr pretrained weights
if cfg.exp_args.testr_ckpt_dir is not None:
ckpt = torch.load(cfg.exp_args.testr_ckpt_dir, map_location="cpu")
load_result = detector.load_state_dict(ckpt['model'], strict=False)
if accelerator.is_main_process:
print("Loaded TESTR checkpoint keys:")
print(" - Missing keys:", load_result.missing_keys)
loaded_models['testr'] = detector.train().to(device)
# -------------------------------- RESUME TRAINING ---------------------------------------
if cfg.exp_args['resume_ckpt_dir'] is not None:
ckpt_dir = f"{cfg.exp_args['resume_ckpt_dir']}"
ckpt=torch.load(ckpt_dir, map_location="cpu")
for model_name, model in loaded_models.items():
if model_name in ckpt:
missing, unexpected = model.load_state_dict(ckpt[model_name], strict=False)
print(f"RESUME TRAINING - Loaded {model_name} | Missing keys: {len(missing)} | Unexpected keys: {len(unexpected)}")
else:
print(f"Warning: No checkpoint found for {model_name}")
for model in loaded_models.values():
model.to(device)
return loaded_models, ckpt_dir
return loaded_models, None
def set_training_params(accelerator, models, cfg):
train_params=[]
all_model_names=[]
train_model_names=[]
for model_name, model in models.items():
for name, param in model.named_parameters():
all_model_names.append(name)
# stage1 training (training the ctrlnet and unet attention layers of the image restoration module)
if cfg.exp_args.finetuning_method == 'image_restoration_module':
if 'controlnet' in name or ('unet' in name and 'attn' in name):
param.requires_grad = True
train_model_names.append(name)
train_params.append(param)
else:
param.requires_grad = False
# stage2 training (training the text spotting module)
elif cfg.exp_args.finetuning_method == 'text_spotting_module':
if 'testr' in name:
param.requires_grad = True
train_model_names.append(name)
train_params.append(param)
else:
param.requires_grad = False
# stage3 training (training both the image restoration and text spotting modules)
elif cfg.exp_args.finetuning_method == 'all_modules':
if 'controlnet' in name or ('unet' in name and 'attn' in name) or ('testr' in name):
param.requires_grad = True
train_model_names.append(name)
train_params.append(param)
else:
param.requires_grad = False
# print modules to be trained
if accelerator.is_main_process:
print('================================================================= MODELS TO BE TRAINED =================================================================')
chunk_size = 10 # Adjust based on readability
for i in range(0, len(train_model_names), chunk_size):
print(train_model_names[i:i+chunk_size]) # Print in smaller chunks
return train_params, train_model_names