How to build a Recurrent Neural Network in TensorFlow (1/7)
Erik Hallström
48825

I tried to use your model for learning a cumulative max function. I changed the num_classes to 1 and loss function to mean squared error. But it seems that the model is not able to learn the simple function even after 100 epochs.

It would be really appreciated if you could guide me what I am doing wrong.

Here is the changed code:

import numpy as np
from random import randint
import tensorflow as tf
from tensorflow.contrib import rnn
import matplotlib.pyplot as plt
# code to generate data which simulates max function
# a length parameter is passed, to generate data for test and training purposes separately
def generateData(length):
x=np.array([randint(0,1000) for p in range(0,length)])
# computes the max till ith position
y = np.maximum.accumulate(x)
x = x.reshape((batch_size, -1))
y = y.reshape((batch_size, -1))
return (x, y)
num_epochs = 100
total_series_length = 60000
truncated_backprop_length = 15
state_size = 10
num_classes = 1
echo_step = 3
batch_size = 5
num_batches = total_series_length//batch_size//truncated_backprop_length
batchX_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length])
batchY_placeholder = tf.placeholder(tf.float32, [batch_size, truncated_backprop_length])
# Define output weights
weights = tf.Variable(np.random.rand(state_size, num_classes), dtype = tf.float32)
biases = tf.Variable(np.zeros((1, num_classes)), dtype = tf.float32)
# Define labels
labels_series = tf.unstack(batchY_placeholder, axis=1)
def RNN(x,weights, biases):
# Prepare data shape to match `rnn` function requirements

# Split into tensors of shape (batch_size, truncated_backprop_length)
inputs_series = tf.split(x, truncated_backprop_length, 1)
    # Define a lstm cell with tensorflow
lstm_cell = rnn.BasicLSTMCell(state_size)
    # Get lstm cell output
outputs, states = rnn.static_rnn(lstm_cell, inputs_series, dtype=tf.float32)

# Linear activation, using rnn inner loop last output
return tf.squeeze([tf.matmul(output, weights) + biases for output in outputs])

pred = RNN(batchX_placeholder, weights, biases)
# give a loss function and optimizer
mseloss = tf.losses.mean_squared_error(pred, labels_series)
optimizer = tf.train.AdagradOptimizer(0.3).minimize(mseloss)
# Initialize the variables
init = tf.global_variables_initializer()
def plot(loss_list):
plt.cla()
plt.plot(loss_list)
plt.draw()
plt.pause(0.0001)
# Launch the graph
with tf.Session() as sess:
sess.run(init)
plt.ion()
plt.figure()
plt.show()
loss_list = []
    x,y = generateData(total_series_length)
for epoch_idx in range(num_epochs):
for batch_idx in range(num_batches):
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
            batchX = x[:,start_idx:end_idx]
batchY = y[:,start_idx:end_idx]
# main function call for running the session which trains the model
_mseloss, _optmimizer, = sess.run(
[mseloss, optimizer],
feed_dict={
batchX_placeholder:batchX,
batchY_placeholder:batchY,
})
            loss_list.append(_mseloss)
# do the plotting
if batch_idx%100 == 0:
print("Step",batch_idx, "Loss", _mseloss)
plot(loss_list)
# training completed, proceed for testing on test data
print("testing data")
test_length = 150
num_batches = test_length//batch_size//truncated_backprop_length
x,y = generateData(test_length)
for batch_idx in range(num_batches):
start_idx = batch_idx * truncated_backprop_length
end_idx = start_idx + truncated_backprop_length
        batchX = x[:,start_idx:end_idx]
batchY = y[:,start_idx:end_idx]
        _mseloss,_pred,_labels = sess.run(
[mseloss,pred,labels_series],
feed_dict={
batchX_placeholder:batchX,
batchY_placeholder:batchY,
})
print("Testing data prediction")
# _pred contains the output of the model on the test data batchX
print(_pred )
# batchY contains the expected output 
print("Target")
print(np.transpose(batchY))
# _mseloss contains mean squared error on the test set
print("%f loss is ", _mseloss)
print("finished")
plt.ioff()
plt.show()
Show your support

Clapping shows how much you appreciated Praneet Khandelwal’s story.