pdfcoffee

Recurrent Neural Networks
# query.shape: (batch_size, num_units)
# values are encoder states at every timestep i
# values.shape: (batch_size, num_timesteps, num_units)
# add time axis to query: (batch_size, 1, num_units)
query_with_time_axis = tf.expand_dims(query, axis=1)
# compute score:
score = self.V(tf.keras.activations.tanh(
self.W1(values) + self.W2(query_with_time_axis)))
# compute softmax
alignment = tf.nn.softmax(score, axis=1)
# compute attended output
context = tf.reduce_sum(
tf.linalg.matmul(
tf.linalg.matrix_transpose(alignment),
values
), axis=1
)
context = tf.expand_dims(context, axis=1)
return context, alignment
The Luong attention is multiplicative, but the general implementation is similar.
Instead of declaring three linear transformations W1, W2, and V, we only have a
single one W. The steps in the call() method follows the same general steps – first,
we compute the scores according to the equation for Luong's attention as described
in the last section, then compute the alignments as the corresponding softmax
version of the scores, then the context vector as the dot product of the alignment
and the values. Like the weights in the Bahdanau attention class, the weight
matrices represented by the dense layer W are learned during training:
class LuongAttention(tf.keras.layers.Layer):
def __init__(self, num_units):
super(LuongAttention, self).__init__()
self.W = tf.keras.layers.Dense(num_units)
def call(self, query, values):
# add time axis to query
query_with_time_axis = tf.expand_dims(query, axis=1)
# compute score
score = tf.linalg.matmul(
query_with_time_axis, self.W(values), transpose_b=True)
# compute softmax
alignment = tf.nn.softmax(score, axis=2)
# compute attended output
context = tf.matmul(alignment, values)
return context, alignment
[ 332 ]