--- title: Title keywords: fastai sidebar: home_sidebar summary: "summary" ---
%load_ext autoreload
%autoreload 2
from collections import defaultdict
import os
import torch
import torch.nn as nn
from htools.ml import BaseModel, GRelu, JRelu, variable_lr_optimizer, stats
os.path.exists('scratch_inheritance_and_mixins.ipynb')
class HNet(BaseModel):
def __init__(self, x_dim, hidden_dim):
super().__init__(locals())
layers = [nn.Linear(x_dim, hidden_dim),
nn.LeakyReLU(),
nn.Linear(hidden_dim, 3),
nn.Softmax(-1)]
self.layers = nn.Sequential(*layers)
def forward(self, x):
return self.layers(x)
hnet = HNet(4, 6)
hnet
x = torch.randint(10, (3, 4), dtype=torch.float)
x
hnet(x)
hnet.weight_stats()
hnet.save(99, '../data')
hnet2 = HNet.from_path('../data/model_e99.pth')
hnet2(x)
hnet2.weight_stats()
hnet3 = HNet.from_path('../data/model_e99_v2.pth', verbose=False)
hnet3.weight_stats()
class Nested(BaseModel):
def __init__(self, x_dim, hidden):
super().__init__(locals())
seq1 = nn.Sequential(nn.Linear(x_dim, hidden),
nn.Dropout(),
nn.LeakyReLU())
seq2 = nn.Sequential(nn.Linear(hidden, 1),
nn.Sigmoid())
self.groups = nn.ModuleList([seq1, seq2])
def forward(self, x):
for group in self.groups:
x = group(x)
return x
nested = Nested(4, 8)
nested
variable_lr_optimizer(nested)
variable_lr_optimizer(groups=nested.groups, lrs=[1, 2])
device = 'gpu' if torch.cuda.is_available() else 'cpu'
device
def train(epochs, train_dl, val_dl, model, criterion, hooks, lr=3e-3,
metrics=None, verbose=True):
for hook in hooks:
model.register_backward_hook(hook)
optim = variable_lr_optimizer()
stats = defaultdict(list)
for epoch in range(epochs):
model.to(device).train()
e_loss = 0.0
e_total = 0
# for i, (x, y) in enumerate(train_dl):
# optim.zero_grad()
# x.to(device)
# y.to(device)
# bs = x.shape[0]
# # Forward pass
# y_hat = model(x)
# loss = criterion(y_hat, y, reduction='mean')
# # Backward pass
# loss.backward()
# optim.step()
# # Update mini batch stats.
# e_total += bs
# loss += loss * bs
# # Evaluate on validation set.
# val_stats = validation_metrics()
# # Update epoch stats.
# stats['loss'].append(e_total)
# stats['val_loss'].append()
# Print epoch stats.
return stats
def gradient_stats_hook(model, grad_in, grad_out):
print(stats(grad_out))
train(3, None, None, hnet, nn.BCEWithLogitsLoss, [gradient_stats_hook])