--- title: Title keywords: fastai sidebar: home_sidebar ---
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import Dataset, DataLoader
from htools import *
class SeqDataset(Dataset):
def __init__(self, sentence_tokens, labels):
self.x = sentence_tokens
self.y = labels
def __getitem__(self, i):
return self.x[i], self.y[i]
def __len__(self):
return len(self.x)
class PositionalAdjustment(nn.Module):
def __init__(self, seq_len, emb_len):
super().__init__()
self.weight = nn.Parameter(torch.Tensor(seq_len, emb_len))
self.weight.data.uniform_(-1, 1)
def forward(self, x):
return x + self.weight
pos = PositionalAdjustment(3, 5)
pos.weight
class SeqNet(nn.Module):
def __init__(self, vocab_size, emb_len, seq_len):
super().__init__()
self.emb = nn.Embedding(vocab_size, emb_len)
self.pos = PositionalAdjustment(seq_len, emb_len)
# self.emb_t = nn.Embedding(seq_len, emb_len)
# self.t_idx = torch.arange(0, seq_len, dtype=torch.long)
def forward(self, x):
# return self.emb(x) + self.emb_t(self.t_idx)
x = self.emb(x)
print(x)
return self.pos(x)
v_size = 5
e_len = 4
s_len = 3
bs = 2
sent_tokens = torch.randint(0, 5, (8, 3))
labels = torch.randint(0, 2, size=(8,))
ds = SeqDataset(sent_tokens, labels)
dl = DataLoader(ds, batch_size=bs, shuffle=False)
x, y = next(iter(dl))
print(x, y)
net = SeqNet(v_size, e_len, s_len)
net
net.emb.weight
net.pos.weight
yhat = net(x)
yhat.shape
yhat
fc = nn.Linear(4, 1)
fc.weight
# First row of item 1 in batch dotted with weight matrix.
-1.2705*.3051 -.2961*4.62 +.4608*-.6619 +.5565*.0259 + fc.bias
yhat @ fc.weight.t() + fc.bias
fc(yhat)
class Dropin(nn.Module):
"""Think about if this would need to work differently in training vs.
eval mode, like multiplicative dropout.
Work in progress, not sure if xavier normal is a good choice - just an
example.
Also look into if floating point addition might be faster/slower on gpu
than multiplication.
"""
def __init__(self, *dims):
super().__init__()
self.weight = nn.Parameter(torch.Tensor(*dims))
nn.init.xavier_normal_(self.weight.data)
def forward(self, x):
return x + self.weight
drop = Dropin(*yhat.shape)
drop.weight
yhat
drop(yhat)
torch.corr?
np.corrcoef(yhat.detach().numpy().flatten(),
drop(yhat).detach().numpy().flatten())
np.corrcoef(drop.weight.detach().numpy().flatten(),
drop(yhat).detach().numpy().flatten())