diff --git a/examples/3_NeuralNetworks/.recurrent_multilayernetwork.py.swp b/examples/3_NeuralNetworks/.recurrent_multilayernetwork.py.swp new file mode 100644 index 00000000..af5db53f Binary files /dev/null and b/examples/3_NeuralNetworks/.recurrent_multilayernetwork.py.swp differ diff --git a/examples/3_NeuralNetworks/autoencoder.py b/examples/3_NeuralNetworks/autoencoder.py index f87f6b23..3a314e07 100644 --- a/examples/3_NeuralNetworks/autoencoder.py +++ b/examples/3_NeuralNetworks/autoencoder.py @@ -83,7 +83,7 @@ def decoder(x): optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(cost) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py b/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py new file mode 100644 index 00000000..c5e32e62 --- /dev/null +++ b/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py @@ -0,0 +1,126 @@ +''' +A Bidirectional Recurrent Neural Network (LSTM) implementation example using TensorFlow library. +This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/) +Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf + +Author: Aymeric Damien +Project: https://github.com/aymericdamien/TensorFlow-Examples/ +''' + +from __future__ import print_function + +import tensorflow as tf +from tensorflow.python.ops import rnn, rnn_cell +import numpy as np +import time +# Import MNIST data +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) + +''' +To classify images using a bidirectional recurrent neural network, we consider +every image row as a sequence of pixels. Because MNIST image shape is 28*28px, +we will then handle 28 sequences of 28 steps for every sample. +''' + +# Parameters +learning_rate = 0.001 +training_iters = 100000 +batch_size = 128 +display_step = 10 + +# Network Parameters +n_input = 28 # MNIST data input (img shape: 28*28) +n_steps = 28 # timesteps +n_hidden = 128 # hidden layer num of features +n_classes = 10 # MNIST total classes (0-9 digits) +n_layers = 2 # Number of Hidden Layers + +# tf Graph input +x = tf.placeholder("float", [None, n_steps, n_input]) +y = tf.placeholder("float", [None, n_classes]) + +# Define weights +weights = { + # Hidden layer weights => 2*n_hidden because of forward + backward cells + 'out': tf.Variable(tf.random_normal([2*n_hidden, n_classes])) +} +biases = { + 'out': tf.Variable(tf.random_normal([n_classes])) +} + + +def BiRNN(x, weights, biases): + + # Prepare data shape to match `bidirectional_rnn` function requirements + # Current data input shape: (batch_size, n_steps, n_input) + # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) + + # Permuting batch_size and n_steps + x = tf.transpose(x, [1, 0, 2]) + # Reshape to (n_steps*batch_size, n_input) + x = tf.reshape(x, [-1, n_input]) + # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) + x = tf.split(0, n_steps, x) + + # Define lstm cells with tensorflow + # Forward direction cell + lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + lstm_fw_cell = rnn_cell.MultiRNNCell([lstm_fw_cell] * n_layers, state_is_tuple=True) + # Backward direction cell + lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + lstm_bw_cell = rnn_cell.MultiRNNCell([lstm_bw_cell] * n_layers, state_is_tuple=True) + + # Get lstm cell output + try: + outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, + dtype=tf.float32) + except Exception: # Old TensorFlow version only returns outputs not states + outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, + dtype=tf.float32) + + # Linear activation, using rnn inner loop last output + return tf.matmul(outputs[-1], weights['out']) + biases['out'] + +pred = BiRNN(x, weights, biases) + +# Define loss and optimizer +cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) +optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) + +# Evaluate model +correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) +accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) + +# Initializing the variables +init = tf.global_variables_initializer() + +# Launch the graph +t = time.time() +with tf.Session() as sess: + sess.run(init) + step = 1 + # Keep training until reach max iterations + while step * batch_size < training_iters: + batch_x, batch_y = mnist.train.next_batch(batch_size) + # Reshape data to get 28 seq of 28 elements + batch_x = batch_x.reshape((batch_size, n_steps, n_input)) + # Run optimization op (backprop) + sess.run(optimizer, feed_dict={x: batch_x, y: batch_y}) + if step % display_step == 0: + # Calculate batch accuracy + acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}) + # Calculate batch loss + loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y}) + print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \ + "{:.6f}".format(loss) + ", Training Accuracy= " + \ + "{:.5f}".format(acc)) + step += 1 + print("Optimization Finished!") + print("Time:%f"%(time.time() - t)) + # Calculate accuracy for 128 mnist test images + test_len = 128 + test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) + test_label = mnist.test.labels[:test_len] + print("Testing Accuracy:", \ + sess.run(accuracy, feed_dict={x: test_data, y: test_label})) diff --git a/examples/3_NeuralNetworks/bidirectional_rnn.py b/examples/3_NeuralNetworks/bidirectional_rnn.py index f8fcf3e5..4e79153b 100644 --- a/examples/3_NeuralNetworks/bidirectional_rnn.py +++ b/examples/3_NeuralNetworks/bidirectional_rnn.py @@ -12,7 +12,7 @@ import tensorflow as tf from tensorflow.python.ops import rnn, rnn_cell import numpy as np - +import time # Import MNIST data from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) @@ -90,9 +90,10 @@ def BiRNN(x, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph +t = time.time() with tf.Session() as sess: sess.run(init) step = 1 @@ -113,7 +114,7 @@ def BiRNN(x, weights, biases): "{:.5f}".format(acc)) step += 1 print("Optimization Finished!") - + print("Time:%f"%(time.time() - t)) # Calculate accuracy for 128 mnist test images test_len = 128 test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) diff --git a/examples/3_NeuralNetworks/convolutional_network.py b/examples/3_NeuralNetworks/convolutional_network.py index 81461237..1f10886f 100644 --- a/examples/3_NeuralNetworks/convolutional_network.py +++ b/examples/3_NeuralNetworks/convolutional_network.py @@ -104,7 +104,7 @@ def conv_net(x, weights, biases, dropout): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/dynamic_rnn.py b/examples/3_NeuralNetworks/dynamic_rnn.py index 9b9443d9..11b952d7 100644 --- a/examples/3_NeuralNetworks/dynamic_rnn.py +++ b/examples/3_NeuralNetworks/dynamic_rnn.py @@ -162,7 +162,7 @@ def dynamicRNN(x, seqlen, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/multilayer_perceptron.py b/examples/3_NeuralNetworks/multilayer_perceptron.py index b5c990f3..9086018a 100644 --- a/examples/3_NeuralNetworks/multilayer_perceptron.py +++ b/examples/3_NeuralNetworks/multilayer_perceptron.py @@ -64,7 +64,7 @@ def multilayer_perceptron(x, weights, biases): optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/recurrent_multilayer_network.py b/examples/3_NeuralNetworks/recurrent_multilayer_network.py new file mode 100644 index 00000000..8e378e92 --- /dev/null +++ b/examples/3_NeuralNetworks/recurrent_multilayer_network.py @@ -0,0 +1,115 @@ +''' +A Recurrent Neural Network (LSTM) implementation example using TensorFlow library. +This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/) +Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf + +Author: Aymeric Damien +Project: https://github.com/aymericdamien/TensorFlow-Examples/ +''' + +from __future__ import print_function + +import tensorflow as tf +from tensorflow.python.ops import rnn, rnn_cell + +# Import MNIST data +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) + +''' +To classify images using a recurrent neural network, we consider every image +row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then +handle 28 sequences of 28 steps for every sample. +''' + +# Parameters +learning_rate = 0.001 +training_iters = 100000 +batch_size = 128 +display_step = 10 + +# Network Parameters +n_input = 28 # MNIST data input (img shape: 28*28) +n_steps = 28 # timesteps +n_hidden = 128 # hidden layer num of features +n_classes = 10 # MNIST total classes (0-9 digits) +n_layers = 2 # Number of Hidden Layers + +# tf Graph input +x = tf.placeholder("float", [None, n_steps, n_input]) +y = tf.placeholder("float", [None, n_classes]) + +# Define weights +weights = { + 'out': tf.Variable(tf.random_normal([n_hidden, n_classes])) +} +biases = { + 'out': tf.Variable(tf.random_normal([n_classes])) +} + + +def RNN(x, weights, biases): + + # Prepare data shape to match `rnn` function requirements + # Current data input shape: (batch_size, n_steps, n_input) + # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) + + # Permuting batch_size and n_steps + x = tf.transpose(x, [1, 0, 2]) + # Reshaping to (n_steps*batch_size, n_input) + x = tf.reshape(x, [-1, n_input]) + # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) + x = tf.split(0, n_steps, x) + + # Define a lstm cell with tensorflow + lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + # Define a multilayer lstm cell + lstm_cell = rnn_cell.MultiRNNCell([lstm_cell] * n_layers, state_is_tuple=True) + + # Get lstm cell output + outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32) + + # Linear activation, using rnn inner loop last output + return tf.matmul(outputs[-1], weights['out']) + biases['out'] + +pred = RNN(x, weights, biases) + +# Define loss and optimizer +cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) +optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) + +# Evaluate model +correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) +accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) + +# Initializing the variables +init = tf.global_variables_initializer() + +# Launch the graph +with tf.Session() as sess: + sess.run(init) + step = 1 + # Keep training until reach max iterations + while step * batch_size < training_iters: + batch_x, batch_y = mnist.train.next_batch(batch_size) + # Reshape data to get 28 seq of 28 elements + batch_x = batch_x.reshape((batch_size, n_steps, n_input)) + # Run optimization op (backprop) + sess.run(optimizer, feed_dict={x: batch_x, y: batch_y}) + if step % display_step == 0: + # Calculate batch accuracy + acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}) + # Calculate batch loss + loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y}) + print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \ + "{:.6f}".format(loss) + ", Training Accuracy= " + \ + "{:.5f}".format(acc)) + step += 1 + print("Optimization Finished!") + + # Calculate accuracy for 128 mnist test images + test_len = 128 + test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) + test_label = mnist.test.labels[:test_len] + print("Testing Accuracy:", \ + sess.run(accuracy, feed_dict={x: test_data, y: test_label})) diff --git a/examples/3_NeuralNetworks/recurrent_network.py b/examples/3_NeuralNetworks/recurrent_network.py index 21744364..9182f2bb 100644 --- a/examples/3_NeuralNetworks/recurrent_network.py +++ b/examples/3_NeuralNetworks/recurrent_network.py @@ -80,7 +80,7 @@ def RNN(x, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb new file mode 100644 index 00000000..34423fdd --- /dev/null +++ b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb @@ -0,0 +1,329 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "'''\n", + "A Bidirectional Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\n", + "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n", + "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n", + "\n", + "Author: Aymeric Damien\n", + "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "from tensorflow.python.ops import rnn, rnn_cell\n", + "import numpy as np\n", + "\n", + "# Import MINST data\n", + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "'''\n", + "To classify images using a bidirectional reccurent neural network, we consider\n", + "every image row as a sequence of pixels. Because MNIST image shape is 28*28px,\n", + "we will then handle 28 sequences of 28 steps for every sample.\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Parameters\n", + "learning_rate = 0.001\n", + "training_iters = 100000\n", + "batch_size = 128\n", + "display_step = 10\n", + "\n", + "# Network Parameters\n", + "n_input = 28 # MNIST data input (img shape: 28*28)\n", + "n_steps = 28 # timesteps\n", + "n_hidden = 128 # hidden layer num of features\n", + "n_classes = 10 # MNIST total classes (0-9 digits)\n", + "\n", + "# tf Graph input\n", + "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n", + "y = tf.placeholder(\"float\", [None, n_classes])\n", + "\n", + "# Define weights\n", + "weights = {\n", + " # Hidden layer weights => 2*n_hidden because of foward + backward cells\n", + " 'out': tf.Variable(tf.random_normal([2*n_hidden, n_classes]))\n", + "}\n", + "biases = {\n", + " 'out': tf.Variable(tf.random_normal([n_classes]))\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBiRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;31m# Define loss and optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mBiRNN\u001b[0;34m(x, weights, biases)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Old TensorFlow version only returns outputs not states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n\u001b[0;32m---> 26\u001b[0;31m dtype=tf.float32)\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Linear activation, using rnn inner loop last output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mbidirectional_rnn\u001b[0;34m(cell_fw, cell_bw, inputs, initial_state_fw, initial_state_bw, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"FW\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfw_scope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m output_fw, output_state_fw = rnn(cell_fw, inputs, initial_state_fw, dtype,\n\u001b[0;32m--> 538\u001b[0;31m sequence_length, scope=fw_scope)\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;31m# Backward direction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mrnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 224\u001b[0m state_size=cell.state_size)\n\u001b[1;32m 225\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mconcat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# i = input_gate, j = new_input, f = forget_gate, o = output_gate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m_linear\u001b[0;34m(args, output_size, bias, bias_start, scope)\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"Linear\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 904\u001b[0m matrix = vs.get_variable(\n\u001b[0;32m--> 905\u001b[0;31m \"Matrix\", [total_arg_size, output_size], dtype=dtype)\n\u001b[0m\u001b[1;32m 906\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 907\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 1022\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 848\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 852\u001b[0m def _get_partitioned_variable(self,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m def _get_partitioned_variable(\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 631\u001b[0m \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 632\u001b[0;31m name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m 633\u001b[0m \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 634\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], + "source": [ + "def BiRNN(x, weights, biases):\n", + "\n", + " # Prepare data shape to match `bidirectional_rnn` function requirements\n", + " # Current data input shape: (batch_size, n_steps, n_input)\n", + " # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)\n", + " \n", + " # Permuting batch_size and n_steps\n", + " x = tf.transpose(x, [1, 0, 2])\n", + " # Reshape to (n_steps*batch_size, n_input)\n", + " x = tf.reshape(x, [-1, n_input])\n", + " # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)\n", + " x = tf.split(0, n_steps, x)\n", + "\n", + " # Define lstm cells with tensorflow\n", + " # Forward direction cell\n", + " lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n", + " # Backward direction cell\n", + " lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n", + "\n", + " # Get lstm cell output\n", + " try:\n", + " outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n", + " dtype=tf.float32)\n", + " except Exception: # Old TensorFlow version only returns outputs not states\n", + " outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n", + " dtype=tf.float32)\n", + "\n", + " # Linear activation, using rnn inner loop last output\n", + " return tf.matmul(outputs[-1], weights['out']) + biases['out']\n", + "\n", + "pred = BiRNN(x, weights, biases)\n", + "\n", + "# Define loss and optimizer\n", + "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n", + "\n", + "# Evaluate model\n", + "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n", + "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", + "\n", + "# Initializing the variables\n", + "init = tf.global_variables_initializer()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 1280, Minibatch Loss= 1.810298, Training Accuracy= 0.41406\n", + "Iter 2560, Minibatch Loss= 1.653038, Training Accuracy= 0.48438\n", + "Iter 3840, Minibatch Loss= 1.088781, Training Accuracy= 0.65625\n", + "Iter 5120, Minibatch Loss= 0.936294, Training Accuracy= 0.67188\n", + "Iter 6400, Minibatch Loss= 0.814716, Training Accuracy= 0.75000\n", + "Iter 7680, Minibatch Loss= 0.643409, Training Accuracy= 0.78906\n", + "Iter 8960, Minibatch Loss= 0.671239, Training Accuracy= 0.78906\n", + "Iter 10240, Minibatch Loss= 0.474204, Training Accuracy= 0.83594\n", + "Iter 11520, Minibatch Loss= 0.427347, Training Accuracy= 0.86719\n", + "Iter 12800, Minibatch Loss= 0.424198, Training Accuracy= 0.85938\n", + "Iter 14080, Minibatch Loss= 0.458741, Training Accuracy= 0.84375\n", + "Iter 15360, Minibatch Loss= 0.467792, Training Accuracy= 0.86719\n", + "Iter 16640, Minibatch Loss= 0.199288, Training Accuracy= 0.94531\n", + "Iter 17920, Minibatch Loss= 0.339153, Training Accuracy= 0.89844\n", + "Iter 19200, Minibatch Loss= 0.307545, Training Accuracy= 0.92188\n", + "Iter 20480, Minibatch Loss= 0.240278, Training Accuracy= 0.90625\n", + "Iter 21760, Minibatch Loss= 0.253847, Training Accuracy= 0.92188\n", + "Iter 23040, Minibatch Loss= 0.202617, Training Accuracy= 0.92969\n", + "Iter 24320, Minibatch Loss= 0.290327, Training Accuracy= 0.91406\n", + "Iter 25600, Minibatch Loss= 0.168713, Training Accuracy= 0.96094\n", + "Iter 26880, Minibatch Loss= 0.156594, Training Accuracy= 0.96875\n", + "Iter 28160, Minibatch Loss= 0.333502, Training Accuracy= 0.88281\n", + "Iter 29440, Minibatch Loss= 0.246582, Training Accuracy= 0.91406\n", + "Iter 30720, Minibatch Loss= 0.231173, Training Accuracy= 0.92188\n", + "Iter 32000, Minibatch Loss= 0.218554, Training Accuracy= 0.91406\n", + "Iter 33280, Minibatch Loss= 0.154140, Training Accuracy= 0.93750\n", + "Iter 34560, Minibatch Loss= 0.202632, Training Accuracy= 0.93750\n", + "Iter 35840, Minibatch Loss= 0.205678, Training Accuracy= 0.92188\n", + "Iter 37120, Minibatch Loss= 0.274195, Training Accuracy= 0.92969\n", + "Iter 38400, Minibatch Loss= 0.236683, Training Accuracy= 0.94531\n", + "Iter 39680, Minibatch Loss= 0.132524, Training Accuracy= 0.96094\n", + "Iter 40960, Minibatch Loss= 0.144923, Training Accuracy= 0.92969\n", + "Iter 42240, Minibatch Loss= 0.102355, Training Accuracy= 0.96875\n", + "Iter 43520, Minibatch Loss= 0.231111, Training Accuracy= 0.94531\n", + "Iter 44800, Minibatch Loss= 0.205104, Training Accuracy= 0.94531\n", + "Iter 46080, Minibatch Loss= 0.166295, Training Accuracy= 0.95312\n", + "Iter 47360, Minibatch Loss= 0.143017, Training Accuracy= 0.96094\n", + "Iter 48640, Minibatch Loss= 0.126032, Training Accuracy= 0.95312\n", + "Iter 49920, Minibatch Loss= 0.226964, Training Accuracy= 0.92188\n", + "Iter 51200, Minibatch Loss= 0.093971, Training Accuracy= 0.96875\n", + "Iter 52480, Minibatch Loss= 0.074533, Training Accuracy= 0.96875\n", + "Iter 53760, Minibatch Loss= 0.099194, Training Accuracy= 0.97656\n", + "Iter 55040, Minibatch Loss= 0.083363, Training Accuracy= 0.96875\n", + "Iter 56320, Minibatch Loss= 0.099111, Training Accuracy= 0.96875\n", + "Iter 57600, Minibatch Loss= 0.172313, Training Accuracy= 0.94531\n", + "Iter 58880, Minibatch Loss= 0.067944, Training Accuracy= 0.98438\n", + "Iter 60160, Minibatch Loss= 0.076590, Training Accuracy= 0.98438\n", + "Iter 61440, Minibatch Loss= 0.122575, Training Accuracy= 0.94531\n", + "Iter 62720, Minibatch Loss= 0.129073, Training Accuracy= 0.96875\n", + "Iter 64000, Minibatch Loss= 0.074188, Training Accuracy= 0.97656\n", + "Iter 65280, Minibatch Loss= 0.079397, Training Accuracy= 0.96094\n", + "Iter 66560, Minibatch Loss= 0.120032, Training Accuracy= 0.96094\n", + "Iter 67840, Minibatch Loss= 0.161353, Training Accuracy= 0.94531\n", + "Iter 69120, Minibatch Loss= 0.124139, Training Accuracy= 0.96875\n", + "Iter 70400, Minibatch Loss= 0.153240, Training Accuracy= 0.95312\n", + "Iter 71680, Minibatch Loss= 0.091048, Training Accuracy= 0.97656\n", + "Iter 72960, Minibatch Loss= 0.106168, Training Accuracy= 0.97656\n", + "Iter 74240, Minibatch Loss= 0.089945, Training Accuracy= 0.96875\n", + "Iter 75520, Minibatch Loss= 0.103590, Training Accuracy= 0.97656\n", + "Iter 76800, Minibatch Loss= 0.037315, Training Accuracy= 0.99219\n", + "Iter 78080, Minibatch Loss= 0.161314, Training Accuracy= 0.96875\n", + "Iter 79360, Minibatch Loss= 0.079335, Training Accuracy= 0.99219\n", + "Iter 80640, Minibatch Loss= 0.054627, Training Accuracy= 0.97656\n", + "Iter 81920, Minibatch Loss= 0.071345, Training Accuracy= 0.97656\n", + "Iter 83200, Minibatch Loss= 0.086784, Training Accuracy= 0.96875\n", + "Iter 84480, Minibatch Loss= 0.047563, Training Accuracy= 0.98438\n", + "Iter 85760, Minibatch Loss= 0.103399, Training Accuracy= 0.98438\n", + "Iter 87040, Minibatch Loss= 0.062650, Training Accuracy= 0.98438\n", + "Iter 88320, Minibatch Loss= 0.024324, Training Accuracy= 0.99219\n", + "Iter 89600, Minibatch Loss= 0.064202, Training Accuracy= 0.98438\n", + "Iter 90880, Minibatch Loss= 0.036536, Training Accuracy= 1.00000\n", + "Iter 92160, Minibatch Loss= 0.025906, Training Accuracy= 0.99219\n", + "Iter 93440, Minibatch Loss= 0.060209, Training Accuracy= 0.97656\n", + "Iter 94720, Minibatch Loss= 0.154609, Training Accuracy= 0.96094\n", + "Iter 96000, Minibatch Loss= 0.068655, Training Accuracy= 0.98438\n", + "Iter 97280, Minibatch Loss= 0.053270, Training Accuracy= 0.98438\n", + "Iter 98560, Minibatch Loss= 0.074571, Training Accuracy= 0.98438\n", + "Iter 99840, Minibatch Loss= 0.065880, Training Accuracy= 0.98438\n", + "Optimization Finished!\n", + "Testing Accuracy: 0.984375\n" + ] + } + ], + "source": [ + "# Launch the graph\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + " step = 1\n", + " # Keep training until reach max iterations\n", + " while step * batch_size < training_iters:\n", + " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", + " # Reshape data to get 28 seq of 28 elements\n", + " batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n", + " # Run optimization op (backprop)\n", + " sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n", + " if step % display_step == 0:\n", + " # Calculate batch accuracy\n", + " acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n", + " # Calculate batch loss\n", + " loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n", + " print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n", + " \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n", + " \"{:.5f}\".format(acc)\n", + " step += 1\n", + " print \"Optimization Finished!\"\n", + "\n", + " # Calculate accuracy for 128 mnist test images\n", + " test_len = 128\n", + " test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n", + " test_label = mnist.test.labels[:test_len]\n", + " print \"Testing Accuracy:\", \\\n", + " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb new file mode 100644 index 00000000..6dee5504 --- /dev/null +++ b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb @@ -0,0 +1,221 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "'''\n", + "A Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\n", + "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n", + "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n", + "\n", + "Author: Aymeric Damien\n", + "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "from tensorflow.python.ops import rnn, rnn_cell\n", + "import numpy as np\n", + "\n", + "# Import MINST data\n", + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\nTo classify images using a reccurent neural network, we consider every image\\nrow as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\\nhandle 28 sequences of 28 steps for every sample.\\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "'''\n", + "To classify images using a reccurent neural network, we consider every image\n", + "row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\n", + "handle 28 sequences of 28 steps for every sample.\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Parameters\n", + "learning_rate = 0.001\n", + "training_iters = 100000\n", + "batch_size = 128\n", + "display_step = 10\n", + "\n", + "# Network Parameters\n", + "n_input = 28 # MNIST data input (img shape: 28*28)\n", + "n_steps = 28 # timesteps\n", + "n_hidden = 128 # hidden layer num of features\n", + "n_classes = 10 # MNIST total classes (0-9 digits)\n", + "\n", + "# tf Graph input\n", + "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n", + "y = tf.placeholder(\"float\", [None, n_classes])\n", + "\n", + "# Define weights\n", + "weights = {\n", + " 'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))\n", + "}\n", + "biases = {\n", + " 'out': tf.Variable(tf.random_normal([n_classes]))\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def RNN(x, weights, biases):\n", + "\n", + " # Prepare data shape to match `rnn` function requirements\n", + " # Current data input shape: (batch_size, n_steps, n_input)\n", + " # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)\n", + " \n", + " # Permuting batch_size and n_steps\n", + " x = tf.transpose(x, [1, 0, 2])\n", + " # Reshaping to (n_steps*batch_size, n_input)\n", + " x = tf.reshape(x, [-1, n_input])\n", + " # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)\n", + " x = tf.split(0, n_steps, x)\n", + "\n", + " # Define a lstm cell with tensorflow\n", + " lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n", + "\n", + " # Get lstm cell output\n", + " outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n", + "\n", + " # Linear activation, using rnn inner loop last output\n", + " return tf.matmul(outputs[-1], weights['out']) + biases['out']\n", + "\n", + "pred = RNN(x, weights, biases)\n", + "\n", + "# Define loss and optimizer\n", + "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n", + "\n", + "# Evaluate model\n", + "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n", + "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", + "\n", + "# Initializing the variables\n", + "init = tf." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Launch the graph\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + " step = 1\n", + " # Keep training until reach max iterations\n", + " while step * batch_size < training_iters:\n", + " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", + " # Reshape data to get 28 seq of 28 elements\n", + " batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n", + " # Run optimization op (backprop)\n", + " sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n", + " if step % display_step == 0:\n", + " # Calculate batch accuracy\n", + " acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n", + " # Calculate batch loss\n", + " loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n", + " print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n", + " \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n", + " \"{:.5f}\".format(acc)\n", + " step += 1\n", + " print \"Optimization Finished!\"\n", + "\n", + " # Calculate accuracy for 128 mnist test images\n", + " test_len = 128\n", + " test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n", + " test_label = mnist.test.labels[:test_len]\n", + " print \"Testing Accuracy:\", \\\n", + " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb b/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb index 581d50e8..ddce4e38 100644 --- a/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb +++ b/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb @@ -3,7 +3,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "'''\n", @@ -19,7 +21,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -34,7 +38,7 @@ ], "source": [ "import tensorflow as tf\n", - "from tensorflow.models.rnn import rnn, rnn_cell\n", + "from tensorflow.python.ops import rnn, rnn_cell\n", "import numpy as np\n", "\n", "# Import MINST data\n", @@ -45,7 +49,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "'''\n", @@ -57,8 +63,10 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "execution_count": 4, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "# Parameters\n", @@ -89,9 +97,34 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBiRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;31m# Define loss and optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mBiRNN\u001b[0;34m(x, weights, biases)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Old TensorFlow version only returns outputs not states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n\u001b[0;32m---> 26\u001b[0;31m dtype=tf.float32)\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Linear activation, using rnn inner loop last output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mbidirectional_rnn\u001b[0;34m(cell_fw, cell_bw, inputs, initial_state_fw, initial_state_bw, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"FW\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfw_scope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m output_fw, output_state_fw = rnn(cell_fw, inputs, initial_state_fw, dtype,\n\u001b[0;32m--> 538\u001b[0;31m sequence_length, scope=fw_scope)\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;31m# Backward direction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mrnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 224\u001b[0m state_size=cell.state_size)\n\u001b[1;32m 225\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mconcat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# i = input_gate, j = new_input, f = forget_gate, o = output_gate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m_linear\u001b[0;34m(args, output_size, bias, bias_start, scope)\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"Linear\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 904\u001b[0m matrix = vs.get_variable(\n\u001b[0;32m--> 905\u001b[0;31m \"Matrix\", [total_arg_size, output_size], dtype=dtype)\n\u001b[0m\u001b[1;32m 906\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 907\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 1022\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 848\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 852\u001b[0m def _get_partitioned_variable(self,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m def _get_partitioned_variable(\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 631\u001b[0m \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 632\u001b[0;31m name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m 633\u001b[0m \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 634\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], "source": [ "def BiRNN(x, weights, biases):\n", "\n", @@ -134,98 +167,25 @@ "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", "\n", "# Initializing the variables\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, + "execution_count": 2, + "metadata": { + "collapsed": false + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iter 1280, Minibatch Loss= 1.689740, Training Accuracy= 0.36719\n", - "Iter 2560, Minibatch Loss= 1.477009, Training Accuracy= 0.44531\n", - "Iter 3840, Minibatch Loss= 1.245874, Training Accuracy= 0.53125\n", - "Iter 5120, Minibatch Loss= 0.990923, Training Accuracy= 0.64062\n", - "Iter 6400, Minibatch Loss= 0.752950, Training Accuracy= 0.71875\n", - "Iter 7680, Minibatch Loss= 1.023025, Training Accuracy= 0.61719\n", - "Iter 8960, Minibatch Loss= 0.921414, Training Accuracy= 0.68750\n", - "Iter 10240, Minibatch Loss= 0.719829, Training Accuracy= 0.75000\n", - "Iter 11520, Minibatch Loss= 0.468657, Training Accuracy= 0.86719\n", - "Iter 12800, Minibatch Loss= 0.654315, Training Accuracy= 0.78125\n", - "Iter 14080, Minibatch Loss= 0.595391, Training Accuracy= 0.83594\n", - "Iter 15360, Minibatch Loss= 0.392862, Training Accuracy= 0.83594\n", - "Iter 16640, Minibatch Loss= 0.421122, Training Accuracy= 0.92188\n", - "Iter 17920, Minibatch Loss= 0.311471, Training Accuracy= 0.88281\n", - "Iter 19200, Minibatch Loss= 0.276949, Training Accuracy= 0.92188\n", - "Iter 20480, Minibatch Loss= 0.170499, Training Accuracy= 0.94531\n", - "Iter 21760, Minibatch Loss= 0.419481, Training Accuracy= 0.86719\n", - "Iter 23040, Minibatch Loss= 0.183765, Training Accuracy= 0.92188\n", - "Iter 24320, Minibatch Loss= 0.386232, Training Accuracy= 0.86719\n", - "Iter 25600, Minibatch Loss= 0.335571, Training Accuracy= 0.88281\n", - "Iter 26880, Minibatch Loss= 0.169092, Training Accuracy= 0.92969\n", - "Iter 28160, Minibatch Loss= 0.247623, Training Accuracy= 0.92969\n", - "Iter 29440, Minibatch Loss= 0.242989, Training Accuracy= 0.94531\n", - "Iter 30720, Minibatch Loss= 0.253811, Training Accuracy= 0.92188\n", - "Iter 32000, Minibatch Loss= 0.169660, Training Accuracy= 0.93750\n", - "Iter 33280, Minibatch Loss= 0.291349, Training Accuracy= 0.90625\n", - "Iter 34560, Minibatch Loss= 0.172026, Training Accuracy= 0.95312\n", - "Iter 35840, Minibatch Loss= 0.186019, Training Accuracy= 0.93750\n", - "Iter 37120, Minibatch Loss= 0.298480, Training Accuracy= 0.89062\n", - "Iter 38400, Minibatch Loss= 0.158750, Training Accuracy= 0.92188\n", - "Iter 39680, Minibatch Loss= 0.162706, Training Accuracy= 0.94531\n", - "Iter 40960, Minibatch Loss= 0.339814, Training Accuracy= 0.86719\n", - "Iter 42240, Minibatch Loss= 0.068817, Training Accuracy= 0.99219\n", - "Iter 43520, Minibatch Loss= 0.188742, Training Accuracy= 0.93750\n", - "Iter 44800, Minibatch Loss= 0.176708, Training Accuracy= 0.92969\n", - "Iter 46080, Minibatch Loss= 0.096726, Training Accuracy= 0.96875\n", - "Iter 47360, Minibatch Loss= 0.220973, Training Accuracy= 0.92969\n", - "Iter 48640, Minibatch Loss= 0.226749, Training Accuracy= 0.94531\n", - "Iter 49920, Minibatch Loss= 0.188906, Training Accuracy= 0.94531\n", - "Iter 51200, Minibatch Loss= 0.145194, Training Accuracy= 0.95312\n", - "Iter 52480, Minibatch Loss= 0.168948, Training Accuracy= 0.95312\n", - "Iter 53760, Minibatch Loss= 0.069116, Training Accuracy= 0.97656\n", - "Iter 55040, Minibatch Loss= 0.228721, Training Accuracy= 0.93750\n", - "Iter 56320, Minibatch Loss= 0.152915, Training Accuracy= 0.95312\n", - "Iter 57600, Minibatch Loss= 0.126974, Training Accuracy= 0.96875\n", - "Iter 58880, Minibatch Loss= 0.078870, Training Accuracy= 0.97656\n", - "Iter 60160, Minibatch Loss= 0.225498, Training Accuracy= 0.95312\n", - "Iter 61440, Minibatch Loss= 0.111760, Training Accuracy= 0.97656\n", - "Iter 62720, Minibatch Loss= 0.161434, Training Accuracy= 0.97656\n", - "Iter 64000, Minibatch Loss= 0.207190, Training Accuracy= 0.94531\n", - "Iter 65280, Minibatch Loss= 0.103831, Training Accuracy= 0.96094\n", - "Iter 66560, Minibatch Loss= 0.153846, Training Accuracy= 0.93750\n", - "Iter 67840, Minibatch Loss= 0.082717, Training Accuracy= 0.96875\n", - "Iter 69120, Minibatch Loss= 0.199301, Training Accuracy= 0.95312\n", - "Iter 70400, Minibatch Loss= 0.139725, Training Accuracy= 0.96875\n", - "Iter 71680, Minibatch Loss= 0.169596, Training Accuracy= 0.95312\n", - "Iter 72960, Minibatch Loss= 0.142444, Training Accuracy= 0.96094\n", - "Iter 74240, Minibatch Loss= 0.145822, Training Accuracy= 0.95312\n", - "Iter 75520, Minibatch Loss= 0.129086, Training Accuracy= 0.94531\n", - "Iter 76800, Minibatch Loss= 0.078082, Training Accuracy= 0.97656\n", - "Iter 78080, Minibatch Loss= 0.151803, Training Accuracy= 0.94531\n", - "Iter 79360, Minibatch Loss= 0.050142, Training Accuracy= 0.98438\n", - "Iter 80640, Minibatch Loss= 0.136788, Training Accuracy= 0.95312\n", - "Iter 81920, Minibatch Loss= 0.130100, Training Accuracy= 0.94531\n", - "Iter 83200, Minibatch Loss= 0.058298, Training Accuracy= 0.98438\n", - "Iter 84480, Minibatch Loss= 0.120124, Training Accuracy= 0.96094\n", - "Iter 85760, Minibatch Loss= 0.064916, Training Accuracy= 0.97656\n", - "Iter 87040, Minibatch Loss= 0.137179, Training Accuracy= 0.93750\n", - "Iter 88320, Minibatch Loss= 0.138268, Training Accuracy= 0.95312\n", - "Iter 89600, Minibatch Loss= 0.072827, Training Accuracy= 0.97656\n", - "Iter 90880, Minibatch Loss= 0.123839, Training Accuracy= 0.96875\n", - "Iter 92160, Minibatch Loss= 0.087194, Training Accuracy= 0.96875\n", - "Iter 93440, Minibatch Loss= 0.083489, Training Accuracy= 0.97656\n", - "Iter 94720, Minibatch Loss= 0.131827, Training Accuracy= 0.95312\n", - "Iter 96000, Minibatch Loss= 0.098764, Training Accuracy= 0.96875\n", - "Iter 97280, Minibatch Loss= 0.115553, Training Accuracy= 0.94531\n", - "Iter 98560, Minibatch Loss= 0.079704, Training Accuracy= 0.96875\n", - "Iter 99840, Minibatch Loss= 0.064562, Training Accuracy= 0.98438\n", - "Optimization Finished!\n", - "Testing Accuracy: 0.992188\n" + "ename": "NameError", + "evalue": "name 'init' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Launch the graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# Keep training until reach max iterations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'init' is not defined" ] } ], @@ -259,6 +219,15 @@ " print \"Testing Accuracy:\", \\\n", " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -270,14 +239,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2.0 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.11" + "version": "2.7.12" } }, "nbformat": 4, diff --git a/notebooks/3_NeuralNetworks/recurrent_network.ipynb b/notebooks/3_NeuralNetworks/recurrent_network.ipynb index 32f3a939..2ccada45 100644 --- a/notebooks/3_NeuralNetworks/recurrent_network.ipynb +++ b/notebooks/3_NeuralNetworks/recurrent_network.ipynb @@ -3,7 +3,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "'''\n", @@ -18,8 +20,10 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": {}, + "execution_count": 8, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -44,9 +48,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\nTo classify images using a reccurent neural network, we consider every image\\nrow as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\\nhandle 28 sequences of 28 steps for every sample.\\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "'''\n", "To classify images using a reccurent neural network, we consider every image\n", @@ -57,8 +74,10 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "execution_count": 9, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "# Parameters\n", @@ -88,9 +107,33 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Variable RNN/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 18, in RNN\n outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n File \"\", line 23, in \n pred = RNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;31m# Define loss and optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mRNN\u001b[0;34m(x, weights, biases)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;31m# Get lstm cell output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlstm_cell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;31m# Linear activation, using rnn inner loop last output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mrnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 224\u001b[0m state_size=cell.state_size)\n\u001b[1;32m 225\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mconcat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# i = input_gate, j = new_input, f = forget_gate, o = output_gate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m_linear\u001b[0;34m(args, output_size, bias, bias_start, scope)\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"Linear\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 904\u001b[0m matrix = vs.get_variable(\n\u001b[0;32m--> 905\u001b[0;31m \"Matrix\", [total_arg_size, output_size], dtype=dtype)\n\u001b[0m\u001b[1;32m 906\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 907\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 1022\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 848\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 852\u001b[0m def _get_partitioned_variable(self,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m def _get_partitioned_variable(\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 631\u001b[0m \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 632\u001b[0;31m name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m 633\u001b[0m \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 634\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Variable RNN/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 18, in RNN\n outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n File \"\", line 23, in \n pred = RNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], "source": [ "def RNN(x, weights, biases):\n", "\n", @@ -125,98 +168,29 @@ "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", "\n", "# Initializing the variables\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, + "execution_count": 10, + "metadata": { + "collapsed": false + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iter 1280, Minibatch Loss= 1.538532, Training Accuracy= 0.49219\n", - "Iter 2560, Minibatch Loss= 1.462834, Training Accuracy= 0.50781\n", - "Iter 3840, Minibatch Loss= 1.048393, Training Accuracy= 0.66406\n", - "Iter 5120, Minibatch Loss= 0.889872, Training Accuracy= 0.71875\n", - "Iter 6400, Minibatch Loss= 0.681855, Training Accuracy= 0.76562\n", - "Iter 7680, Minibatch Loss= 0.987207, Training Accuracy= 0.69531\n", - "Iter 8960, Minibatch Loss= 0.759543, Training Accuracy= 0.71094\n", - "Iter 10240, Minibatch Loss= 0.557055, Training Accuracy= 0.80469\n", - "Iter 11520, Minibatch Loss= 0.371352, Training Accuracy= 0.89844\n", - "Iter 12800, Minibatch Loss= 0.661293, Training Accuracy= 0.80469\n", - "Iter 14080, Minibatch Loss= 0.474259, Training Accuracy= 0.86719\n", - "Iter 15360, Minibatch Loss= 0.328436, Training Accuracy= 0.88281\n", - "Iter 16640, Minibatch Loss= 0.348017, Training Accuracy= 0.93750\n", - "Iter 17920, Minibatch Loss= 0.340086, Training Accuracy= 0.88281\n", - "Iter 19200, Minibatch Loss= 0.261532, Training Accuracy= 0.89844\n", - "Iter 20480, Minibatch Loss= 0.161785, Training Accuracy= 0.94531\n", - "Iter 21760, Minibatch Loss= 0.419619, Training Accuracy= 0.83594\n", - "Iter 23040, Minibatch Loss= 0.120714, Training Accuracy= 0.95312\n", - "Iter 24320, Minibatch Loss= 0.339519, Training Accuracy= 0.89062\n", - "Iter 25600, Minibatch Loss= 0.405463, Training Accuracy= 0.88281\n", - "Iter 26880, Minibatch Loss= 0.172193, Training Accuracy= 0.95312\n", - "Iter 28160, Minibatch Loss= 0.256769, Training Accuracy= 0.91406\n", - "Iter 29440, Minibatch Loss= 0.247753, Training Accuracy= 0.91406\n", - "Iter 30720, Minibatch Loss= 0.230820, Training Accuracy= 0.91406\n", - "Iter 32000, Minibatch Loss= 0.216861, Training Accuracy= 0.93750\n", - "Iter 33280, Minibatch Loss= 0.236337, Training Accuracy= 0.89062\n", - "Iter 34560, Minibatch Loss= 0.252351, Training Accuracy= 0.93750\n", - "Iter 35840, Minibatch Loss= 0.180090, Training Accuracy= 0.92188\n", - "Iter 37120, Minibatch Loss= 0.304125, Training Accuracy= 0.91406\n", - "Iter 38400, Minibatch Loss= 0.114474, Training Accuracy= 0.96094\n", - "Iter 39680, Minibatch Loss= 0.158405, Training Accuracy= 0.96875\n", - "Iter 40960, Minibatch Loss= 0.285858, Training Accuracy= 0.92188\n", - "Iter 42240, Minibatch Loss= 0.134199, Training Accuracy= 0.96094\n", - "Iter 43520, Minibatch Loss= 0.235847, Training Accuracy= 0.92969\n", - "Iter 44800, Minibatch Loss= 0.155971, Training Accuracy= 0.94531\n", - "Iter 46080, Minibatch Loss= 0.061549, Training Accuracy= 0.99219\n", - "Iter 47360, Minibatch Loss= 0.232569, Training Accuracy= 0.94531\n", - "Iter 48640, Minibatch Loss= 0.270348, Training Accuracy= 0.91406\n", - "Iter 49920, Minibatch Loss= 0.202416, Training Accuracy= 0.92188\n", - "Iter 51200, Minibatch Loss= 0.113857, Training Accuracy= 0.96094\n", - "Iter 52480, Minibatch Loss= 0.137900, Training Accuracy= 0.94531\n", - "Iter 53760, Minibatch Loss= 0.052416, Training Accuracy= 0.98438\n", - "Iter 55040, Minibatch Loss= 0.312064, Training Accuracy= 0.91406\n", - "Iter 56320, Minibatch Loss= 0.144335, Training Accuracy= 0.93750\n", - "Iter 57600, Minibatch Loss= 0.114723, Training Accuracy= 0.96875\n", - "Iter 58880, Minibatch Loss= 0.193597, Training Accuracy= 0.96094\n", - "Iter 60160, Minibatch Loss= 0.110877, Training Accuracy= 0.95312\n", - "Iter 61440, Minibatch Loss= 0.119864, Training Accuracy= 0.96094\n", - "Iter 62720, Minibatch Loss= 0.118780, Training Accuracy= 0.94531\n", - "Iter 64000, Minibatch Loss= 0.082259, Training Accuracy= 0.97656\n", - "Iter 65280, Minibatch Loss= 0.087364, Training Accuracy= 0.97656\n", - "Iter 66560, Minibatch Loss= 0.207975, Training Accuracy= 0.92969\n", - "Iter 67840, Minibatch Loss= 0.120612, Training Accuracy= 0.96875\n", - "Iter 69120, Minibatch Loss= 0.070608, Training Accuracy= 0.96875\n", - "Iter 70400, Minibatch Loss= 0.100786, Training Accuracy= 0.96094\n", - "Iter 71680, Minibatch Loss= 0.114746, Training Accuracy= 0.94531\n", - "Iter 72960, Minibatch Loss= 0.083427, Training Accuracy= 0.96875\n", - "Iter 74240, Minibatch Loss= 0.089978, Training Accuracy= 0.96094\n", - "Iter 75520, Minibatch Loss= 0.195322, Training Accuracy= 0.94531\n", - "Iter 76800, Minibatch Loss= 0.161109, Training Accuracy= 0.96094\n", - "Iter 78080, Minibatch Loss= 0.169762, Training Accuracy= 0.94531\n", - "Iter 79360, Minibatch Loss= 0.054240, Training Accuracy= 0.98438\n", - "Iter 80640, Minibatch Loss= 0.160100, Training Accuracy= 0.95312\n", - "Iter 81920, Minibatch Loss= 0.110728, Training Accuracy= 0.96875\n", - "Iter 83200, Minibatch Loss= 0.054918, Training Accuracy= 0.98438\n", - "Iter 84480, Minibatch Loss= 0.104170, Training Accuracy= 0.96875\n", - "Iter 85760, Minibatch Loss= 0.071871, Training Accuracy= 0.97656\n", - "Iter 87040, Minibatch Loss= 0.170529, Training Accuracy= 0.96094\n", - "Iter 88320, Minibatch Loss= 0.087350, Training Accuracy= 0.96875\n", - "Iter 89600, Minibatch Loss= 0.079943, Training Accuracy= 0.96875\n", - "Iter 90880, Minibatch Loss= 0.128451, Training Accuracy= 0.92969\n", - "Iter 92160, Minibatch Loss= 0.046963, Training Accuracy= 0.98438\n", - "Iter 93440, Minibatch Loss= 0.162998, Training Accuracy= 0.96875\n", - "Iter 94720, Minibatch Loss= 0.122588, Training Accuracy= 0.96094\n", - "Iter 96000, Minibatch Loss= 0.073954, Training Accuracy= 0.97656\n", - "Iter 97280, Minibatch Loss= 0.130790, Training Accuracy= 0.96094\n", - "Iter 98560, Minibatch Loss= 0.067689, Training Accuracy= 0.97656\n", - "Iter 99840, Minibatch Loss= 0.186411, Training Accuracy= 0.92188\n", - "Optimization Finished!\n", - "Testing Accuracy: 0.976562\n" + "ename": "InvalidArgumentError", + "evalue": "You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\nCaused by op u'Placeholder', defined at:\n File \"/usr/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n \"__main__\", fname, loader, pkg_name)\n File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n exec code in run_globals\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py\", line 3, in \n app.launch_new_instance()\n File \"/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py\", line 474, in start\n ioloop.IOLoop.instance().start()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py\", line 160, in start\n super(ZMQIOLoop, self).start()\n File \"/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py\", line 887, in start\n handler_func(fd_obj, events)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 433, in _handle_events\n self._handle_recv()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 465, in _handle_recv\n self._run_callback(callback, msg)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 407, in _run_callback\n callback(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n handler(stream, idents, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n user_expressions, allow_stdin)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n if self.run_code(code, result):\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 14, in \n x = tf.placeholder(\"float\", [None, n_steps, n_input])\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py\", line 1587, in placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 2043, in _placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py\", line 759, in apply_op\n op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 2240, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 1128, in __init__\n self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mbatch_x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_input\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;31m# Run optimization op (backprop)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_y\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdisplay_step\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;31m# Calculate batch accuracy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 765\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 767\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 768\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 962\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 963\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 965\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 966\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1012\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1013\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1015\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1016\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1032\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1033\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1036\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mInvalidArgumentError\u001b[0m: You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\nCaused by op u'Placeholder', defined at:\n File \"/usr/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n \"__main__\", fname, loader, pkg_name)\n File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n exec code in run_globals\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py\", line 3, in \n app.launch_new_instance()\n File \"/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py\", line 474, in start\n ioloop.IOLoop.instance().start()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py\", line 160, in start\n super(ZMQIOLoop, self).start()\n File \"/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py\", line 887, in start\n handler_func(fd_obj, events)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 433, in _handle_events\n self._handle_recv()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 465, in _handle_recv\n self._run_callback(callback, msg)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 407, in _run_callback\n callback(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n handler(stream, idents, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n user_expressions, allow_stdin)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n if self.run_code(code, result):\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 14, in \n x = tf.placeholder(\"float\", [None, n_steps, n_input])\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py\", line 1587, in placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 2043, in _placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py\", line 759, in apply_op\n op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 2240, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 1128, in __init__\n self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n" ] } ], @@ -250,6 +224,15 @@ " print \"Testing Accuracy:\", \\\n", " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -261,14 +244,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2.0 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.11" + "version": "2.7.12" } }, "nbformat": 4,