import numpy as np
class RecurrentNeuralNetwork:
#input (word), expected output (next word), num of words (num of recurrences), array expected outputs, learning rate
def __init__ (self, xs, ys, rl, eo, lr):
#initial input (first word)
self.x = np.zeros(xs)
#input size
self.xs = xs
#expected output (next word)
self.y = np.zeros(ys)
#output size
self.ys = ys
#weight matrix for interpreting results from LSTM cell (num words x num words matrix)
self.w = np.random.random((ys, ys))
#matrix used in RMSprop
self.G = np.zeros_like(self.w)
#length of the recurrent network - number of recurrences i.e num of words
self.rl = rl
#learning rate
self.lr = lr
#array for storing inputs
self.ia = np.zeros((rl+1,xs))
#array for storing cell states
self.ca = np.zeros((rl+1,ys))
#array for storing outputs
self.oa = np.zeros((rl+1,ys))
#array for storing hidden states
self.ha = np.zeros((rl+1,ys))
#forget gate
self.af = np.zeros((rl+1,ys))
#input gate
self.ai = np.zeros((rl+1,ys))
#cell state
self.ac = np.zeros((rl+1,ys))
#output gate
self.ao = np.zeros((rl+1,ys))
#array of expected output values
self.eo = np.vstack((np.zeros(eo.shape[0]), eo.T))
#declare LSTM cell (input, output, amount of recurrence, learning rate)
self.LSTM = LSTM(xs, ys, rl, lr)
#activation function. simple nonlinearity, convert nums into probabilities between 0 and 1
def sigmoid(self, x):
return 1 / (1 + np.exp(-x))
#the derivative of the sigmoid function. used to compute gradients for backpropagation
def dsigmoid(self, x):
return self.sigmoid(x) * (1 - self.sigmoid(x))
#lets apply a series of matrix operations to our input (curr word) to compute a predicted output (next word)
def forwardProp(self):
for i in range(1, self.rl+1):
self.LSTM.x = np.hstack((self.ha[i-1], self.x))
cs, hs, f, c, o = self.LSTM.forwardProp()
#store computed cell state
self.ca[i] = cs
self.ha[i] = hs
self.af[i] = f
self.ai[i] = inp
self.ac[i] = c
self.ao[i] = o
self.oa[i] = self.sigmoid(np.dot(self.w, hs))
self.x = self.eo[i-1]
return self.oa
def backProp(self):
#update our weight matrices (Both in our Recurrent network, as well as the weight matrices inside LSTM cell)
#init an empty error value
totalError = 0
#initialize matrices for gradient updates
#First, these are RNN level gradients
#cell state
dfcs = np.zeros(self.ys)
#hidden state,
dfhs = np.zeros(self.ys)
#weight matrix
tu = np.zeros((self.ys,self.ys))
#Next, these are LSTM level gradients
#forget gate
tfu = np.zeros((self.ys, self.xs+self.ys))
#input gate
tiu = np.zeros((self.ys, self.xs+self.ys))
#cell unit
tcu = np.zeros((self.ys, self.xs+self.ys))
#output gate
tou = np.zeros((self.ys, self.xs+self.ys))
#loop backwards through recurrences
for i in range(self.rl, -1, -1):
#error = calculatedOutput - expectedOutput
error = self.oa[i] - self.eo[i]
#calculate update for weight matrix
#(error * derivative of the output) * hidden state
tu += np.dot(np.atleast_2d(error * self.dsigmoid(self.oa[i])), np.atleast_2d(self.ha[i]).T)
#Time to propagate error back to exit of LSTM cell
#1. error * RNN weight matrix
error = np.dot(error, self.w)
&
