Illustration RNN Encoder-Decoder with input and output
In this post, I will illustrate recurrent neural network (RNN) Encoder-Decoder with the input and output. The model architecture is based on the official tutorial of Pytorch.
Illustration of input and output:
bmm
: The context vector `c_i = \sum_{j=1}^{T_x} \alpha_{ij}h_j` in paper Bahdanau et al.
In above figure, for time step `t` of decoder:
$$c_t = [\alpha_{t1}, \alpha_{t2},..., \alpha_{tL}]_{1\times L} \times \left[ \begin{array}{l}
h_1\\
h_2\\
h_3\\
h_L
\end{array} \right]_{L \times H}$$
Code
import torch
import torch.nn as nn
from torch.autograd import Variable
from torch import optim
import torch.nn.functional as F
from seq2seq.data_helper import use_cuda,MAX_LENGTH
class EncoderRNN(nn.Module):
def __init__(self, input_size, hidden_size, embed_size, n_layers=1):
super(EncoderRNN, self).__init__()
self.n_layers = n_layers
self.hidden_size = hidden_size
self.embedding = nn.Embedding(input_size, embed_size)
self.gru = nn.GRU( embed_size,hidden_size)
def forward(self, input, hidden):
embedded = self.embedding(input).view(1, 1, -1)
output = embedded
for i in range(self.n_layers):
output, hidden = self.gru(output, hidden)
return output, hidden
def initHidden(self):
result = Variable(torch.zeros(1, 1, self.hidden_size))
if use_cuda:
return result.cuda()
else:
return result
class AttnDecoderRNN(nn.Module):
def __init__(self, hidden_size, output_size, embed_size, n_layers=1, dropout_p=0.1, max_length=MAX_LENGTH):
super(AttnDecoderRNN, self).__init__()
self.hidden_size = hidden_size
self.output_size = output_size
self.n_layers = n_layers
self.dropout_p = dropout_p
self.max_length = max_length
self.embed_size = embed_size
self.embedding = nn.Embedding(self.output_size, self.embed_size)
self.attn = nn.Linear(self.hidden_size + self.embed_size, self.max_length)
self.attn_combine = nn.Linear(self.hidden_size + self.embed_size, self.hidden_size)
self.dropout = nn.Dropout(self.dropout_p)
self.gru = nn.GRU(self.hidden_size, self.hidden_size)
self.out = nn.Linear(self.hidden_size, self.output_size)
def forward(self, input, hidden, encoder_output, encoder_outputs):
embedded = self.embedding(input).view(1, 1, -1)
embedded = self.dropout(embedded)
attn_weights = F.softmax(
self.attn(torch.cat((embedded[0], hidden[0]), 1)))
attn_applied = torch.bmm(attn_weights.unsqueeze(0),
encoder_outputs.unsqueeze(0))
output = torch.cat((embedded[0], attn_applied[0]), 1)
output = self.attn_combine(output).unsqueeze(0)
for i in range(self.n_layers):
output = F.relu(output)
output, hidden = self.gru(output, hidden)
output = F.log_softmax(self.out(output[0]))
return output, hidden, attn_weights
def initHidden(self):
result = Variable(torch.zeros(1, 1, self.hidden_size))
if use_cuda:
return result.cuda()
else:
return result