-
Notifications
You must be signed in to change notification settings - Fork 14
/
Copy pathtrain.py
86 lines (64 loc) · 3.03 KB
/
train.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
import os
import argparse
from dataset import *
from model import *
from tqdm import tqdm
import time
import datetime
import torch.optim as optim
import torch
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
parser = argparse.ArgumentParser()
parser.add_argument
parser.add_argument('--model_dir', default='./tmp/model10.ckpt', type=str, help='where to save models' )
parser.add_argument('--seed', default=0, type=int, help='random seed')
parser.add_argument('--batch_size', default=16, type=int)
parser.add_argument('--num_slots', default=7, type=int, help='Number of slots in Slot Attention.')
parser.add_argument('--num_iterations', default=3, type=int, help='Number of attention iterations.')
parser.add_argument('--hid_dim', default=64, type=int, help='hidden dimension size')
parser.add_argument('--learning_rate', default=0.0004, type=float)
parser.add_argument('--warmup_steps', default=10000, type=int, help='Number of warmup steps for the learning rate.')
parser.add_argument('--decay_rate', default=0.5, type=float, help='Rate for the learning rate decay.')
parser.add_argument('--decay_steps', default=100000, type=int, help='Number of steps for the learning rate decay.')
parser.add_argument('--num_workers', default=4, type=int, help='number of workers for loading data')
parser.add_argument('--num_epochs', default=1000, type=int, help='number of workers for loading data')
opt = parser.parse_args()
resolution = (128, 128)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
train_set = CLEVR('train')
model = SlotAttentionAutoEncoder(resolution, opt.num_slots, opt.num_iterations, opt.hid_dim).to(device)
# model.load_state_dict(torch.load('./tmp/model6.ckpt')['model_state_dict'])
criterion = nn.MSELoss()
params = [{'params': model.parameters()}]
train_dataloader = torch.utils.data.DataLoader(train_set, batch_size=opt.batch_size,
shuffle=True, num_workers=opt.num_workers)
optimizer = optim.Adam(params, lr=opt.learning_rate)
start = time.time()
i = 0
for epoch in range(opt.num_epochs):
model.train()
total_loss = 0
for sample in tqdm(train_dataloader):
i += 1
if i < opt.warmup_steps:
learning_rate = opt.learning_rate * (i / opt.warmup_steps)
else:
learning_rate = opt.learning_rate
learning_rate = learning_rate * (opt.decay_rate ** (
i / opt.decay_steps))
optimizer.param_groups[0]['lr'] = learning_rate
image = sample['image'].to(device)
recon_combined, recons, masks, slots = model(image)
loss = criterion(recon_combined, image)
total_loss += loss.item()
del recons, masks, slots
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_loss /= len(train_dataloader)
print ("Epoch: {}, Loss: {}, Time: {}".format(epoch, total_loss,
datetime.timedelta(seconds=time.time() - start)))
if not epoch % 10:
torch.save({
'model_state_dict': model.state_dict(),
}, opt.model_dir)