Daniel Voigt Godoy - Deep Learning with PyTorch Step-by-Step A Beginner’s Guide-leanpub

6 self.encoder = encoder
7 self.mlp = nn.Linear(self.d_model, n_outputs)
8
9 self.embed = embedding_layer 1
10 self.cls_token = nn.Parameter(
11 torch.zeros(1, 1, self.d_model)
12 )
13
14 def preprocess(self, X):
15 # N, L -> N, L, D
16 src = self.embed(X)
17 # Special classifier token
18 # 1, 1, D -> N, 1, D
19 cls_tokens = self.cls_token.expand(X.size(0), -1, -1)
20 # Concatenates CLS tokens -> N, 1 + L, D
21 src = torch.cat((cls_tokens, src), dim=1)
22 return src
23
24 def encode(self, source, source_mask=None):
25 # Encoder generates "hidden states"
26 states = self.encoder(source, source_mask) 2
27 # Gets state from first token: CLS
28 cls_state = states[:, 0] # N, 1, D
29 return cls_state
30
31 @staticmethod
32 def source_mask(X): 2
33 cls_mask = torch.ones(X.size(0), 1).type_as(X)
34 pad_mask = torch.cat((cls_mask, X > 0), dim=1).bool()
35 return pad_mask.unsqueeze(1)
36
37 def forward(self, X):
38 src = self.preprocess(X)
39 # Featurizer
40 cls_state = self.encode(src,
41 self.source_mask(X)) 2
42 # Classifier
43 out = self.mlp(cls_state) # N, 1, outputs
44 return out
1 The embedding layer is an argument now.
2 The encoder receives a source mask to flag the padded (and classification)
Word Embeddings | 943