pdfcoffee
Chapter 8decoder_in = tf.expand_dims(tf.constant([word2idx_fr["BOS"]]), axis=0)pred_sent_fr = []while True:decoder_pred, decoder_state = decoder(decoder_in, decoder_state)decoder_pred = tf.argmax(decoder_pred, axis=-1)pred_word = idx2word_fr[decoder_pred.numpy()[0][0]]pred_sent_fr.append(pred_word)if pred_word == "EOS":breakdecoder_in = decoder_predprint("predicted: ", " ".join(pred_sent_fr))def evaluate_bleu_score(encoder, decoder, test_dataset,word2idx_fr, idx2word_fr):bleu_scores = []smooth_fn = SmoothingFunction()for encoder_in, decoder_in, decoder_out in test_dataset:encoder_state = encoder.init_state(batch_size)encoder_out, encoder_state = encoder(encoder_in, encoder_state)decoder_state = encoder_statedecoder_pred, decoder_state = decoder(decoder_in, decoder_state)# compute argmaxdecoder_out = decoder_out.numpy()decoder_pred = tf.argmax(decoder_pred, axis=-1).numpy()for i in range(decoder_out.shape[0]):ref_sent = [idx2word_fr[j] for j indecoder_out[i].tolist() if j > 0]hyp_sent = [idx2word_fr[j] for j indecoder_pred[i].tolist() if j > 0]# remove trailing EOSref_sent = ref_sent[0:-1]hyp_sent = hyp_sent[0:-1]bleu_score = sentence_bleu([ref_sent], hyp_sent,smoothing_function=smooth_fn.method1)bleu_scores.append(bleu_score)[ 325 ]
Recurrent Neural Networksreturn np.mean(np.array(bleu_scores))The training loop is shown as follows. We will use the Adam optimizer for ourmodel. We also set up a checkpoint so we can save our model after every 10 epochs.We then train the model for 250 epochs, and print out the loss, an example sentenceand its translation, and the BLEU score computed over the entire test set:optimizer = tf.keras.optimizers.Adam()checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")checkpoint = tf.train.Checkpoint(optimizer=optimizer,encoder=encoder,decoder=decoder)num_epochs = 250eval_scores = []for e in range(num_epochs):encoder_state = encoder.init_state(batch_size)for batch, data in enumerate(train_dataset):encoder_in, decoder_in, decoder_out = data# print(encoder_in.shape, decoder_in.shape, decoder_out.shape)loss = train_step(encoder_in, decoder_in, decoder_out, encoder_state)print("Epoch: {}, Loss: {:.4f}".format(e + 1, loss.numpy()))if e % 10 == 0:checkpoint.save(file_prefix=checkpoint_prefix)predict(encoder, decoder, batch_size, sents_en, data_en,sents_fr_out, word2idx_fr, idx2word_fr)eval_score = evaluate_bleu_score(encoder, decoder,test_dataset, word2idx_fr, idx2word_fr)print("Eval Score (BLEU): {:.3e}".format(eval_score))# eval_scores.append(eval_score)checkpoint.save(file_prefix=checkpoint_prefix)The results from the first 5 and last 5 epochs of training are shown as follows. Noticethat the loss has gone down from about 1.5 to around 0.07 in epoch 247. The BLEUscores have also gone up by around 2.5 times. Most impressive, however, is thedifference in translation quality between the first 5 and last 5 epochs:[ 326 ]
- Page 309 and 310: Word EmbeddingsOnce trained, we sav
- Page 311 and 312: Word Embeddings4. Pennington, J., S
- Page 313 and 314: Word Embeddings34. Google Research,
- Page 315 and 316: Recurrent Neural NetworksWe will th
- Page 317 and 318: Recurrent Neural NetworksFor notati
- Page 319 and 320: Recurrent Neural NetworksThis probl
- Page 321 and 322: Recurrent Neural NetworksThe line a
- Page 323 and 324: Recurrent Neural NetworksGated recu
- Page 325 and 326: Recurrent Neural NetworksThis probl
- Page 327 and 328: Recurrent Neural NetworksThe topolo
- Page 329 and 330: Recurrent Neural Networkstexts = do
- Page 331 and 332: Recurrent Neural Networksdef call(s
- Page 333 and 334: Recurrent Neural Networks# callback
- Page 335 and 336: Recurrent Neural NetworksExample
- Page 337 and 338: Recurrent Neural NetworksAs can be
- Page 339 and 340: Recurrent Neural Networksdata_dir =
- Page 341 and 342: Recurrent Neural NetworksWe can als
- Page 343 and 344: Recurrent Neural NetworksIn order t
- Page 345 and 346: Recurrent Neural Networkssource_voc
- Page 347 and 348: Recurrent Neural NetworksFinally, w
- Page 349 and 350: Recurrent Neural Networks38 - val_l
- Page 351 and 352: Recurrent Neural NetworksIf you wou
- Page 353 and 354: Recurrent Neural NetworksExample
- Page 355 and 356: Recurrent Neural NetworksNext we ha
- Page 357 and 358: Recurrent Neural Networksself.embed
- Page 359: Recurrent Neural NetworksThis is a
- Page 363 and 364: Recurrent Neural NetworksAttention
- Page 365 and 366: Recurrent Neural NetworksFinally, V
- Page 367 and 368: Recurrent Neural Networks# query.sh
- Page 369 and 370: Recurrent Neural Networksself.atten
- Page 371 and 372: Recurrent Neural Networks30 try to
- Page 373 and 374: Recurrent Neural Networks3. Because
- Page 375 and 376: Recurrent Neural NetworksSummaryIn
- Page 377 and 378: Recurrent Neural Networks18. Shi, X
- Page 380 and 381: AutoencodersAutoencoders are feed-f
- Page 382 and 383: Depending upon the actual dimension
- Page 384 and 385: • __init__(): Here, you define al
- Page 386 and 387: Chapter 9And then we reshape the te
- Page 388 and 389: Chapter 9plt.imshow(x_test[index].r
- Page 390 and 391: Chapter 9Keeping the rest of the co
- Page 392 and 393: noise = np.random.normal(loc=0.5, s
- Page 394 and 395: Chapter 9x_train,validation_data=(x
- Page 396 and 397: Chapter 9import matplotlib.pyplot a
- Page 398 and 399: Chapter 9self.conv4 = Conv2D(1, 3,
- Page 400 and 401: Chapter 9You can see that the image
- Page 402 and 403: [ 367 ]Chapter 9Let us use the prec
- Page 404 and 405: Chapter 9Our autoencoder model take
- Page 406 and 407: We train the autoencoder for 20 epo
- Page 408 and 409: Chapter 90.97905576229095460.989323
Chapter 8
decoder_in = tf.expand_dims(
tf.constant([word2idx_fr["BOS"]]), axis=0)
pred_sent_fr = []
while True:
decoder_pred, decoder_state = decoder(
decoder_in, decoder_state)
decoder_pred = tf.argmax(decoder_pred, axis=-1)
pred_word = idx2word_fr[decoder_pred.numpy()[0][0]]
pred_sent_fr.append(pred_word)
if pred_word == "EOS":
break
decoder_in = decoder_pred
print("predicted: ", " ".join(pred_sent_fr))
def evaluate_bleu_score(encoder, decoder, test_dataset,
word2idx_fr, idx2word_fr):
bleu_scores = []
smooth_fn = SmoothingFunction()
for encoder_in, decoder_in, decoder_out in test_dataset:
encoder_state = encoder.init_state(batch_size)
encoder_out, encoder_state = encoder(encoder_in, encoder_state)
decoder_state = encoder_state
decoder_pred, decoder_state = decoder(
decoder_in, decoder_state)
# compute argmax
decoder_out = decoder_out.numpy()
decoder_pred = tf.argmax(decoder_pred, axis=-1).numpy()
for i in range(decoder_out.shape[0]):
ref_sent = [idx2word_fr[j] for j in
decoder_out[i].tolist() if j > 0]
hyp_sent = [idx2word_fr[j] for j in
decoder_pred[i].tolist() if j > 0]
# remove trailing EOS
ref_sent = ref_sent[0:-1]
hyp_sent = hyp_sent[0:-1]
bleu_score = sentence_bleu([ref_sent], hyp_sent,
smoothing_function=smooth_fn.method1)
bleu_scores.append(bleu_score)
[ 325 ]
Change language