Concise Lecture Notes - Lesson 7 | Fastai v3 (2019)

These notes were typed out by me while watching the lecture, for a quick revision later on. To be able to fully understand them, they should be used alongside the jupyter notebooks that are available here:




def conv(ni,nf): return nn.Conv2d(ni, nf, kernel_size=3, stride=2, padding=1)

model = nn.Sequential(
    conv(1, 8), # 14
    conv(8, 16), # 7
    conv(16, 32), # 4
    conv(32, 16), # 2
    conv(16, 10), # 1
    Flatten()     # remove (1,1) grid

How to improve this?

class ResBlock(nn.Module):
    def __init__(self, nf:int):
        super(ResBlock, self).__init__()  #initialize
        self.conv1 = conv_layer(nf, nf, 3, 1) #input and output channels are of same size
        self.conv2 = conv_layer(nf, nf, 3, 1) #input and output channels are of same size
    def forward(self, x:torch.Tensor) -> torch.Tensor:
        return self.conv2(self.conv1(x)) + x #SKIP CONNECTION


nearest neighbour interpolation

Why U-nets work so well

But rather than adding a skip connection that skipped every two convolutions, they added a skip connection from the same part of the downsampling path to the same-sized bit in the upsampling path.

Generative modelling


loss_critic = AdaptiveLoss(nn.BCEWithLogitsLoss())
def create_critic_learner(data, metrics):
    return Learner(data, gan_critic(), metrics=metrics, loss_func=loss_critic, wd=wd)
learn_critic = create_critic_learner(data_crit, accuracy_thresh_expand)
switcher = partial(AdaptiveGANSwitcher, critic_thresh=0.65)
learn = GANLearner.from_learners(learn_gen, learn_crit, weights_gen=(1.,50.),
    show_img=False, switcher=switcher, opt_func=partial(optim.Adam, betas=(0.,0.99)), 

learn.callback_fns.append(partial(GANDiscriminativeLR, mult_lr=5.))

Feature Loss

“in this part of that 28 by 28 grid, is there something that looks kind of furry? Or is there something that looks kind of shiny? Or is there something that was kind of circular? Is there something that kind of looks like an eyeball?”

class FeatureLoss(nn.Module):
    def __init__(self, m_feat, layer_ids, layer_wgts):
        self.m_feat = m_feat
        self.loss_features = [self.m_feat[i] for i in layer_ids]
        self.hooks = hook_outputs(self.loss_features, detach=False)
        self.wgts = layer_wgts
        self.metric_names = ['pixel',] + [f'feat_{i}' for i in range(len(layer_ids))
              ] + [f'gram_{i}' for i in range(len(layer_ids))]

    def make_features(self, x, clone=False):
        return [(o.clone() if clone else o) for o in self.hooks.stored]
    def forward(self, input, target):
        out_feat = self.make_features(target, clone=True)
        in_feat = self.make_features(input)
        self.feat_losses = [base_loss(input,target)]
        self.feat_losses += [base_loss(f_in, f_out)*w
                             for f_in, f_out, w in zip(in_feat, out_feat, self.wgts)]
        self.feat_losses += [base_loss(gram_matrix(f_in), gram_matrix(f_out))*w**2 * 5e3
                             for f_in, f_out, w in zip(in_feat, out_feat, self.wgts)]
        self.metrics = dict(zip(self.metric_names, self.feat_losses))
        return sum(self.feat_losses)
    def __del__(self): self.hooks.remove()
feat_loss = FeatureLoss(vgg_m, blocks[2:5], [5,15,2])

learn = unet_learner(data, arch, wd = 1e-3, loss_func=feat_loss, 
    callback_fns=LossMetrics, blur=True, norm_type=NormType.Weight)


  1. Grab word 1 as an input.
  2. Chuck it through an embedding, create some activations.
  3. Pass that through a matrix product and nonlinearity.
  4. Grab the second word.
  5. Put it through an embedding.
  6. Then we could either add those two things together or concatenate them. Generally speaking, when you see two sets of activations coming together in a diagram, you normally have a choice of concatenate or or add. And that’s going to create the second bunch of activations.
  7. Repeat for word 3
  8. Then you can put it through one more fully connected layer and softmax to create an output.
class Model(nn.Module):
    def __init__(self):
        self.i_h = nn.Embedding(nv,nh)  # green arrow
        self.h_h = nn.Linear(nh,nh)     # brown arrow
        self.h_o = nn.Linear(nh,nv)     # blue arrow = nn.BatchNorm1d(nh)
    def forward(self, x):
        h = torch.zeros(x.shape[1], nh).to(device=x.device)
        for xi in x:
            h += self.i_h(xi)
            h  =
        return self.h_o(h)
class Model(nn.Module):
    def __init__(self):
        self.i_h = nn.Embedding(nv,nh)
        self.h_h = nn.Linear(nh,nh)
        self.h_o = nn.Linear(nh,nv) = nn.BatchNorm1d(nh)
        self.h = torch.zeros(x.shape[1], nh).cuda()
    def forward(self, x):
        res = []
        h = self.h
        for xi in x:
            h = h + self.i_h(xi)
            h = F.relu(self.h_h(h))
        self.h = h.detach()
        res = torch.stack(res)
        res = self.h_o(
        return res
class Model(nn.Module):
    def __init__(self):
        self.i_h = nn.Embedding(nv,nh)
        self.rnn = nn.RNN(nh,nh,2)
        self.h_o = nn.Linear(nh,nv) = nn.BatchNorm1d(nh)
        self.h = torch.zeros(2, x.shape[1], nh).cuda()
    def forward(self, x):
        res,h = self.rnn(self.i_h(x), self.h)
        self.h = h.detach()
        return self.h_o(
comments powered by Disqus