Daniel Voigt Godoy - Deep Learning with PyTorch Step-by-Step A Beginner’s Guide-leanpub

Model Configuration
1 class EncoderSelfAttn(nn.Module):
2 def __init__(self, n_heads, d_model,
3 ff_units, n_features=None):
4 super().__init__()
5 self.n_heads = n_heads
6 self.d_model = d_model
7 self.ff_units = ff_units
8 self.n_features = n_features
9 self.self_attn_heads = \
10 MultiHeadAttention(n_heads,
11 d_model,
12 input_dim=n_features)
13 self.ffn = nn.Sequential(
14 nn.Linear(d_model, ff_units),
15 nn.ReLU(),
16 nn.Linear(ff_units, d_model),
17 )
18
19 def forward(self, query, mask=None):
20 self.self_attn_heads.init_keys(query)
21 att = self.self_attn_heads(query, mask)
22 out = self.ffn(att)
23 return out
24
25 class DecoderSelfAttn(nn.Module):
26 def __init__(self, n_heads, d_model,
27 ff_units, n_features=None):
28 super().__init__()
29 self.n_heads = n_heads
30 self.d_model = d_model
31 self.ff_units = ff_units
32 self.n_features = d_model if n_features is None \
33 else n_features
34 self.self_attn_heads = \
35 MultiHeadAttention(n_heads,
36 d_model,
37 input_dim=self.n_features)
38 self.cross_attn_heads = \
39 MultiHeadAttention(n_heads, d_model)
40 self.ffn = nn.Sequential(
41 nn.Linear(d_model, ff_units),
Putting It All Together | 789