pdfcoffee

Recurrent Neural Networks
Finally, we declare the model with some parameters, then compile it with the Adam
optimizer, the categorical cross-entropy loss function, and accuracy as the metric:
class POSTaggingModel(tf.keras.Model):
def __init__(self, source_vocab_size, target_vocab_size,
embedding_dim, max_seqlen, rnn_output_dim, **kwargs):
super(POSTaggingModel, self).__init__(**kwargs)
self.embed = tf.keras.layers.Embedding(
source_vocab_size, embedding_dim, input_length=max_seqlen)
self.dropout = tf.keras.layers.SpatialDropout1D(0.2)
self.rnn = tf.keras.layers.Bidirectional(
tf.keras.layers.GRU(rnn_output_dim, return_sequences=True))
self.dense = tf.keras.layers.TimeDistributed(
tf.keras.layers.Dense(target_vocab_size))
self.activation = tf.keras.layers.Activation("softmax")
def call(self, x):
x = self.embed(x)
x = self.dropout(x)
x = self.rnn(x)
x = self.dense(x)
x = self.activation(x)
return x
embedding_dim = 128
rnn_output_dim = 256
model = POSTaggingModel(source_vocab_size, target_vocab_size,
embedding_dim, max_seqlen, rnn_output_dim)
model.build(input_shape=(batch_size, max_seqlen))
model.summary()
model.compile(
loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy", masked_accuracy()])
Observant readers might have noticed an additional masked_accuracy() metric
next to the accuracy metric in the preceding code snippet. Because of the padding,
there are a lot of zeros on both the label and prediction, as a result of which the
accuracy numbers are very optimistic. In fact, the validation accuracy reported at the
end of the very first epoch is 0.9116. However, the quality of POS tags generated are
very poor.
[ 312 ]