From 37560127ee64ea942b8cdd69de0516a01769601c Mon Sep 17 00:00:00 2001 From: ajeetksingh Date: Sat, 14 Jan 2017 10:03:36 +0530 Subject: [PATCH 1/2] added multilayer variants of bidirectional and vanill lstm networks --- .../.recurrent_multilayernetwork.py.swp | Bin 0 -> 20480 bytes .../bidirectional_multilayer_rnn.py | 126 +++++++ .../3_NeuralNetworks/bidirectional_rnn.py | 5 +- .../recurrent_multilayer_network.py | 114 ++++++ .../recurrent_multilayernetwork.py | 114 ++++++ .../bidirectional_rnn-checkpoint.ipynb | 329 ++++++++++++++++++ .../recurrent_network-checkpoint.ipynb | 221 ++++++++++++ .../3_NeuralNetworks/bidirectional_rnn.ipynb | 163 ++++----- .../3_NeuralNetworks/recurrent_network.ipynb | 181 +++++----- 9 files changed, 1055 insertions(+), 198 deletions(-) create mode 100644 examples/3_NeuralNetworks/.recurrent_multilayernetwork.py.swp create mode 100644 examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py create mode 100644 examples/3_NeuralNetworks/recurrent_multilayer_network.py create mode 100644 examples/3_NeuralNetworks/recurrent_multilayernetwork.py create mode 100644 notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb create mode 100644 notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb diff --git a/examples/3_NeuralNetworks/.recurrent_multilayernetwork.py.swp b/examples/3_NeuralNetworks/.recurrent_multilayernetwork.py.swp new file mode 100644 index 0000000000000000000000000000000000000000..af5db53fe83701979db7f614db7698fd886c7f97 GIT binary patch literal 20480 zcmeI4U2Gjk700J7-=P6f@q~mXcER^1&R*Mr#JSX;1~? zZ0y@8ltLfE8!F|asDSvW5Kl-vP(UCAML-_P18otN`hrvtFNu(ND4@{rKQpt}*CuUg zg+!&yO27E-?#!8)GiT1s&KzZV;?5(pd|&VZhU=Y-J^P*h-Ct&$Z9B}ExK~O|&tKs> z8fPm<<0O&ltk2B zQhf8h2D}Em2D}Em2D}Em2D}Em2Cgm*=;AtdKbm~4Yxa%q?=@TgzQNsNTfkiJcYo`2 z*SY&QxxcGh-n(1>$7{fAz-z#3z-z#3z-z#3z-z#3z-z#3z-z#3;Qycj(P8ZT+Ze01 zp>Ul4*Wdq_-oe;+!69&QJ7X8X@4=VA=fOiD0jpphjDiuc9XxeCV_yIdgU^A3;BRkd z?C0R8;3wdF;1TcuD8Nau1P+6DgG;E-Z^5I0>UKNmfB;-ZeP09@z#qXkz}LaU;3PN! z7QrXL47drr_*TY#21=m7QP2P{yoIsffd@ecM4%1+{${)b&w;OjuYw5d032Kc7H&?094G~4njhq`vDc#kdjPnI0P-wQo4@3yr z5K(`IkKoOTO{KZI9??>nx20AMs@KkCp`%_{#`j4lZWhAKLw*cpTuBy{&1ZNi%Q8?^ z)`;>_kMpjSov;%}dgjb}n_IE#FnPfn%OYAW zR97Xc^gLi}$T<3g6YCbiMZac;E zF8z{;OcCI-q4Pnf*&lrrpTeAVpvlC7Os?zDyc(*6t@BzhPNY@fPFH0zDSi_-Sz(=F z4AU#EDo6QIi6K$H;#GzKBd(-~dOQ{DoRS*t*s9r%aKmTN1u#?HWNWeRaor8(VT>4m zjV2eRvCK9b<;O&oL8R_UzM@3YV~jGPRmecUVv(Sdp-7VOv{12FMo3hniHt|;mo%&> z``|Ugw44=MF-mr-NQS8tSp!ldQEH3_1WrXdEbwxi%IYS$(0iuPJHxN60C}ns31t;( z1#NGbOp{HM&pqmoPa^9mzxA|8DjF_n-jNAo10%GOY$qM)h}3x`sTzqCihM%|eae|l zcg-M$4&bNk$|p81%%=v((!3yO5?mLsW(-Y@i@!i29q3PFF}WKE8iCG z${L3;LuKUWkRt$%q||8`$s`%)sCq~D3pq2?xvt>bFv=(;r<9u{5As`?MpSUKDC6kZ z{KD)(ByVJ4FYa_?2CtGUjMtEwHZvJa*ikmhoo2;Q5q1Qa*gn|Vnr+3pH3>MfST`ny z9L0JqX`WuyZUOQz%*ic`IhQiVwGmm;L*seKO$~b<>u^k~U1_z8f_BhV1E;i97ik@m z8B#e%TWMUELle*-ht1%m!mP6r@r#lSW)Wi1oT-7#<5H!SCYc+xGdAbMTXm2#JXOVH zd!{YzE3~uLz$-Nt^Og;aw2sxNp_aNuMW%sUS~bd#BOFsGX|vnX;GJRWa8>uf(C*<~ zxGl4qP`1|AOS6vbzSO{JV;OYT^o1D~Z5ljxI+sSy1v>2HX_(O>BWWyVVFbg+h+D#V zwZjJnbLQ2Fs7-Z~FgK!&w1K(0hm6sTg4$qujbtxRI`B{<#yMgcvpAP$l9V;58gmx# z(#f;X#HG?Go#LZ>u1c3Nuwk*bZW!GZwfbMBD%qf9NSmC1xY@ZQ^9vXvS|r?6iZ>>9 z@5c&(TwW77Tsg>q`E+zsu{^6%s!CT1xHC!ejXh9ZDJ>R_(mcQmYeepaE!NE1_IV>t zS85BJn%+4zO_PBO)aA#85~-xQ6)WmZD;G3^4GjuYM^?|db~-jKoxsI5O$~TF5ll=> zqL^XvV`F2i2SdV4Z`To(%p$l*G?MOVG?U-r^0ZWaUYZc&fT?_q=43$&50in?7xB7G z%7EW2BT<#osX9)@3RV%c&f#$hp=fJirU|}=fM=SCGq#Z1&WO8xZZee4vkOcdE@qi$ zCYTw2Of)l18a&FgGKR&u>Y1#Jn#9Z+2W>Xm4oUG?Z3!%9@hdDPCbyB6PKy@Zw_vHT z)3BH8nFUqJChMv^bPly6XgmVYedl3&T3M`4YfW>tgVU0~H)s`Mp@Nw0TeK zqHMP)XuMY~2T`83L_d`(j3Na(19X6;+!buu1tXCM~|ua`@$WywZFDewxq9 zG>2sr=wGC!vQSh9OTpUuf~X9lv4^jj0J`91Tfe&6>I3nja%MuCHDnp_5?PQnTE>s&T`9iRjbJkV8*D8$wtNx zMqGoY>n5qwuAl8XzOIg~%SNm{X4YjTR6j6^rtkm9@Xh`#pznYBe5LD2eE0ttq~Igq zy@1mEfztWQYrt#3Yrt#3Yrt#3Yrt#3Yrt#3Yrt#3YvA>w0q(BB@l1V8vh{!rd+OTk zteB%gbC-x^nViK=B*}0Gl?zBlySfM5X-gaXF7`xoBMY`{-0>6=2S!S{*tYLwwq(Sa zJF}OzDY4x+8cMuzzGio>w`|?oy49j-+iYsu$R9q%teb2$!DvI!?)y+K>=s~at7P>3 z{{+sRFXJqlzW)!q{=1Ct{fpp-;IrTySOcqI7ia?76Zi}E1O5#D1fB=K2EPL5z!C5- zoc&({&w%fP2fzui9bCjY{{`?%@M+Kj0k{D?gLD3;!B@cD;4Ux?_JQ|+r*NkKAovVW zU?2D=&h!5Po&%4AAAoOxbD#n}upev#PvVUK5pWj>z(t(pp9kLr9|wEEU$GhTZEzCo z1vdiPAThdnz1X?0hu47DfY-ol)WFaT=ML%Z?y(OW@?k^05Jz2C443j@LvF3*!-h=s z435x!*w9rSHZ)8BCl`XKOw`cgAhKwQ@8or~gNtAa<1*A0jw;=8HO)YS^!Sje&yjs3 kl8;0}*oBWodTo(N6u7i`&j2B62=eLj+bHtXMtv~$Zvo8j#sB~S literal 0 HcmV?d00001 diff --git a/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py b/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py new file mode 100644 index 00000000..25259769 --- /dev/null +++ b/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py @@ -0,0 +1,126 @@ +''' +A Bidirectional Recurrent Neural Network (LSTM) implementation example using TensorFlow library. +This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/) +Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf + +Author: Aymeric Damien +Project: https://github.com/aymericdamien/TensorFlow-Examples/ +''' + +from __future__ import print_function + +import tensorflow as tf +from tensorflow.python.ops import rnn, rnn_cell +import numpy as np +import time +# Import MNIST data +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) + +''' +To classify images using a bidirectional recurrent neural network, we consider +every image row as a sequence of pixels. Because MNIST image shape is 28*28px, +we will then handle 28 sequences of 28 steps for every sample. +''' + +# Parameters +learning_rate = 0.001 +training_iters = 100000 +batch_size = 128 +display_step = 10 + +# Network Parameters +n_input = 28 # MNIST data input (img shape: 28*28) +n_steps = 28 # timesteps +n_hidden = 128 # hidden layer num of features +n_classes = 10 # MNIST total classes (0-9 digits) +n_layers = 2 # Number of Hidden Layers + +# tf Graph input +x = tf.placeholder("float", [None, n_steps, n_input]) +y = tf.placeholder("float", [None, n_classes]) + +# Define weights +weights = { + # Hidden layer weights => 2*n_hidden because of forward + backward cells + 'out': tf.Variable(tf.random_normal([2*n_hidden, n_classes])) +} +biases = { + 'out': tf.Variable(tf.random_normal([n_classes])) +} + + +def BiRNN(x, weights, biases): + + # Prepare data shape to match `bidirectional_rnn` function requirements + # Current data input shape: (batch_size, n_steps, n_input) + # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) + + # Permuting batch_size and n_steps + x = tf.transpose(x, [1, 0, 2]) + # Reshape to (n_steps*batch_size, n_input) + x = tf.reshape(x, [-1, n_input]) + # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) + x = tf.split(0, n_steps, x) + + # Define lstm cells with tensorflow + # Forward direction cell + lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + lstm_fw_cell = rnn_cell.MultiRNNCell([lstm_fw_cell] * n_layers, state_is_tuple=True) + # Backward direction cell + lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + lstm_bw_cell = rnn_cell.MultiRNNCell([lstm_bw_cell] * n_layers, state_is_tuple=True) + + # Get lstm cell output + try: + outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, + dtype=tf.float32) + except Exception: # Old TensorFlow version only returns outputs not states + outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, + dtype=tf.float32) + + # Linear activation, using rnn inner loop last output + return tf.matmul(outputs[-1], weights['out']) + biases['out'] + +pred = BiRNN(x, weights, biases) + +# Define loss and optimizer +cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) +optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) + +# Evaluate model +correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) +accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) + +# Initializing the variables +init = tf.initialize_all_variables() + +# Launch the graph +t = time.time() +with tf.Session() as sess: + sess.run(init) + step = 1 + # Keep training until reach max iterations + while step * batch_size < training_iters: + batch_x, batch_y = mnist.train.next_batch(batch_size) + # Reshape data to get 28 seq of 28 elements + batch_x = batch_x.reshape((batch_size, n_steps, n_input)) + # Run optimization op (backprop) + sess.run(optimizer, feed_dict={x: batch_x, y: batch_y}) + if step % display_step == 0: + # Calculate batch accuracy + acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}) + # Calculate batch loss + loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y}) + print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \ + "{:.6f}".format(loss) + ", Training Accuracy= " + \ + "{:.5f}".format(acc)) + step += 1 + print("Optimization Finished!") + print("Time:%f"%(time.time() - t)) + # Calculate accuracy for 128 mnist test images + test_len = 128 + test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) + test_label = mnist.test.labels[:test_len] + print("Testing Accuracy:", \ + sess.run(accuracy, feed_dict={x: test_data, y: test_label})) diff --git a/examples/3_NeuralNetworks/bidirectional_rnn.py b/examples/3_NeuralNetworks/bidirectional_rnn.py index f8fcf3e5..cadcdfd2 100644 --- a/examples/3_NeuralNetworks/bidirectional_rnn.py +++ b/examples/3_NeuralNetworks/bidirectional_rnn.py @@ -12,7 +12,7 @@ import tensorflow as tf from tensorflow.python.ops import rnn, rnn_cell import numpy as np - +import time # Import MNIST data from tensorflow.examples.tutorials.mnist import input_data mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) @@ -93,6 +93,7 @@ def BiRNN(x, weights, biases): init = tf.initialize_all_variables() # Launch the graph +t = time.time() with tf.Session() as sess: sess.run(init) step = 1 @@ -113,7 +114,7 @@ def BiRNN(x, weights, biases): "{:.5f}".format(acc)) step += 1 print("Optimization Finished!") - + print("Time:%f"%(time.time() - t)) # Calculate accuracy for 128 mnist test images test_len = 128 test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) diff --git a/examples/3_NeuralNetworks/recurrent_multilayer_network.py b/examples/3_NeuralNetworks/recurrent_multilayer_network.py new file mode 100644 index 00000000..8cb0d8e7 --- /dev/null +++ b/examples/3_NeuralNetworks/recurrent_multilayer_network.py @@ -0,0 +1,114 @@ +''' +A Recurrent Neural Network (LSTM) implementation example using TensorFlow library. +This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/) +Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf + +Author: Aymeric Damien +Project: https://github.com/aymericdamien/TensorFlow-Examples/ +''' + +from __future__ import print_function + +import tensorflow as tf +from tensorflow.python.ops import rnn, rnn_cell + +# Import MNIST data +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) + +''' +To classify images using a recurrent neural network, we consider every image +row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then +handle 28 sequences of 28 steps for every sample. +''' + +# Parameters +learning_rate = 0.001 +training_iters = 100000 +batch_size = 128 +display_step = 10 + +# Network Parameters +n_input = 28 # MNIST data input (img shape: 28*28) +n_steps = 28 # timesteps +n_hidden = 128 # hidden layer num of features +n_classes = 10 # MNIST total classes (0-9 digits) +n_layers = 2 # Number of hidden layers + +# tf Graph input +x = tf.placeholder("float", [None, n_steps, n_input]) +y = tf.placeholder("float", [None, n_classes]) + +# Define weights +weights = { + 'out': tf.Variable(tf.random_normal([n_hidden, n_classes])) +} +biases = { + 'out': tf.Variable(tf.random_normal([n_classes])) +} + + +def RNN(x, weights, biases): + + # Prepare data shape to match `rnn` function requirements + # Current data input shape: (batch_size, n_steps, n_input) + # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) + + # Permuting batch_size and n_steps + x = tf.transpose(x, [1, 0, 2]) + # Reshaping to (n_steps*batch_size, n_input) + x = tf.reshape(x, [-1, n_input]) + # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) + x = tf.split(0, n_steps, x) + + # Define a lstm cell with tensorflow + lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + lstm_cell = rnn_cell.MultiRNNCell([lstm_cell] * n_layers, state_is_tuple=True) + + # Get lstm cell output + outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32) + + # Linear activation, using rnn inner loop last output + return tf.matmul(outputs[-1], weights['out']) + biases['out'] + +pred = RNN(x, weights, biases) + +# Define loss and optimizer +cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) +optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) + +# Evaluate model +correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) +accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) + +# Initializing the variables +init = tf.initialize_all_variables() + +# Launch the graph +with tf.Session() as sess: + sess.run(init) + step = 1 + # Keep training until reach max iterations + while step * batch_size < training_iters: + batch_x, batch_y = mnist.train.next_batch(batch_size) + # Reshape data to get 28 seq of 28 elements + batch_x = batch_x.reshape((batch_size, n_steps, n_input)) + # Run optimization op (backprop) + sess.run(optimizer, feed_dict={x: batch_x, y: batch_y}) + if step % display_step == 0: + # Calculate batch accuracy + acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}) + # Calculate batch loss + loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y}) + print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \ + "{:.6f}".format(loss) + ", Training Accuracy= " + \ + "{:.5f}".format(acc)) + step += 1 + print("Optimization Finished!") + + # Calculate accuracy for 128 mnist test images + test_len = 128 + test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) + test_label = mnist.test.labels[:test_len] + print("Testing Accuracy:", \ + sess.run(accuracy, feed_dict={x: test_data, y: test_label})) diff --git a/examples/3_NeuralNetworks/recurrent_multilayernetwork.py b/examples/3_NeuralNetworks/recurrent_multilayernetwork.py new file mode 100644 index 00000000..8cb0d8e7 --- /dev/null +++ b/examples/3_NeuralNetworks/recurrent_multilayernetwork.py @@ -0,0 +1,114 @@ +''' +A Recurrent Neural Network (LSTM) implementation example using TensorFlow library. +This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/) +Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf + +Author: Aymeric Damien +Project: https://github.com/aymericdamien/TensorFlow-Examples/ +''' + +from __future__ import print_function + +import tensorflow as tf +from tensorflow.python.ops import rnn, rnn_cell + +# Import MNIST data +from tensorflow.examples.tutorials.mnist import input_data +mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) + +''' +To classify images using a recurrent neural network, we consider every image +row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then +handle 28 sequences of 28 steps for every sample. +''' + +# Parameters +learning_rate = 0.001 +training_iters = 100000 +batch_size = 128 +display_step = 10 + +# Network Parameters +n_input = 28 # MNIST data input (img shape: 28*28) +n_steps = 28 # timesteps +n_hidden = 128 # hidden layer num of features +n_classes = 10 # MNIST total classes (0-9 digits) +n_layers = 2 # Number of hidden layers + +# tf Graph input +x = tf.placeholder("float", [None, n_steps, n_input]) +y = tf.placeholder("float", [None, n_classes]) + +# Define weights +weights = { + 'out': tf.Variable(tf.random_normal([n_hidden, n_classes])) +} +biases = { + 'out': tf.Variable(tf.random_normal([n_classes])) +} + + +def RNN(x, weights, biases): + + # Prepare data shape to match `rnn` function requirements + # Current data input shape: (batch_size, n_steps, n_input) + # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) + + # Permuting batch_size and n_steps + x = tf.transpose(x, [1, 0, 2]) + # Reshaping to (n_steps*batch_size, n_input) + x = tf.reshape(x, [-1, n_input]) + # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) + x = tf.split(0, n_steps, x) + + # Define a lstm cell with tensorflow + lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + lstm_cell = rnn_cell.MultiRNNCell([lstm_cell] * n_layers, state_is_tuple=True) + + # Get lstm cell output + outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32) + + # Linear activation, using rnn inner loop last output + return tf.matmul(outputs[-1], weights['out']) + biases['out'] + +pred = RNN(x, weights, biases) + +# Define loss and optimizer +cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) +optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) + +# Evaluate model +correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) +accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) + +# Initializing the variables +init = tf.initialize_all_variables() + +# Launch the graph +with tf.Session() as sess: + sess.run(init) + step = 1 + # Keep training until reach max iterations + while step * batch_size < training_iters: + batch_x, batch_y = mnist.train.next_batch(batch_size) + # Reshape data to get 28 seq of 28 elements + batch_x = batch_x.reshape((batch_size, n_steps, n_input)) + # Run optimization op (backprop) + sess.run(optimizer, feed_dict={x: batch_x, y: batch_y}) + if step % display_step == 0: + # Calculate batch accuracy + acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}) + # Calculate batch loss + loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y}) + print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \ + "{:.6f}".format(loss) + ", Training Accuracy= " + \ + "{:.5f}".format(acc)) + step += 1 + print("Optimization Finished!") + + # Calculate accuracy for 128 mnist test images + test_len = 128 + test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) + test_label = mnist.test.labels[:test_len] + print("Testing Accuracy:", \ + sess.run(accuracy, feed_dict={x: test_data, y: test_label})) diff --git a/notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb new file mode 100644 index 00000000..34423fdd --- /dev/null +++ b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/bidirectional_rnn-checkpoint.ipynb @@ -0,0 +1,329 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "'''\n", + "A Bidirectional Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\n", + "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n", + "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n", + "\n", + "Author: Aymeric Damien\n", + "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "from tensorflow.python.ops import rnn, rnn_cell\n", + "import numpy as np\n", + "\n", + "# Import MINST data\n", + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "'''\n", + "To classify images using a bidirectional reccurent neural network, we consider\n", + "every image row as a sequence of pixels. Because MNIST image shape is 28*28px,\n", + "we will then handle 28 sequences of 28 steps for every sample.\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Parameters\n", + "learning_rate = 0.001\n", + "training_iters = 100000\n", + "batch_size = 128\n", + "display_step = 10\n", + "\n", + "# Network Parameters\n", + "n_input = 28 # MNIST data input (img shape: 28*28)\n", + "n_steps = 28 # timesteps\n", + "n_hidden = 128 # hidden layer num of features\n", + "n_classes = 10 # MNIST total classes (0-9 digits)\n", + "\n", + "# tf Graph input\n", + "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n", + "y = tf.placeholder(\"float\", [None, n_classes])\n", + "\n", + "# Define weights\n", + "weights = {\n", + " # Hidden layer weights => 2*n_hidden because of foward + backward cells\n", + " 'out': tf.Variable(tf.random_normal([2*n_hidden, n_classes]))\n", + "}\n", + "biases = {\n", + " 'out': tf.Variable(tf.random_normal([n_classes]))\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBiRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;31m# Define loss and optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mBiRNN\u001b[0;34m(x, weights, biases)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Old TensorFlow version only returns outputs not states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n\u001b[0;32m---> 26\u001b[0;31m dtype=tf.float32)\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Linear activation, using rnn inner loop last output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mbidirectional_rnn\u001b[0;34m(cell_fw, cell_bw, inputs, initial_state_fw, initial_state_bw, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"FW\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfw_scope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m output_fw, output_state_fw = rnn(cell_fw, inputs, initial_state_fw, dtype,\n\u001b[0;32m--> 538\u001b[0;31m sequence_length, scope=fw_scope)\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;31m# Backward direction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mrnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 224\u001b[0m state_size=cell.state_size)\n\u001b[1;32m 225\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mconcat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# i = input_gate, j = new_input, f = forget_gate, o = output_gate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m_linear\u001b[0;34m(args, output_size, bias, bias_start, scope)\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"Linear\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 904\u001b[0m matrix = vs.get_variable(\n\u001b[0;32m--> 905\u001b[0;31m \"Matrix\", [total_arg_size, output_size], dtype=dtype)\n\u001b[0m\u001b[1;32m 906\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 907\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 1022\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 848\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 852\u001b[0m def _get_partitioned_variable(self,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m def _get_partitioned_variable(\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 631\u001b[0m \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 632\u001b[0;31m name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m 633\u001b[0m \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 634\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], + "source": [ + "def BiRNN(x, weights, biases):\n", + "\n", + " # Prepare data shape to match `bidirectional_rnn` function requirements\n", + " # Current data input shape: (batch_size, n_steps, n_input)\n", + " # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)\n", + " \n", + " # Permuting batch_size and n_steps\n", + " x = tf.transpose(x, [1, 0, 2])\n", + " # Reshape to (n_steps*batch_size, n_input)\n", + " x = tf.reshape(x, [-1, n_input])\n", + " # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)\n", + " x = tf.split(0, n_steps, x)\n", + "\n", + " # Define lstm cells with tensorflow\n", + " # Forward direction cell\n", + " lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n", + " # Backward direction cell\n", + " lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n", + "\n", + " # Get lstm cell output\n", + " try:\n", + " outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n", + " dtype=tf.float32)\n", + " except Exception: # Old TensorFlow version only returns outputs not states\n", + " outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n", + " dtype=tf.float32)\n", + "\n", + " # Linear activation, using rnn inner loop last output\n", + " return tf.matmul(outputs[-1], weights['out']) + biases['out']\n", + "\n", + "pred = BiRNN(x, weights, biases)\n", + "\n", + "# Define loss and optimizer\n", + "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n", + "\n", + "# Evaluate model\n", + "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n", + "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", + "\n", + "# Initializing the variables\n", + "init = tf.global_variables_initializer()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Iter 1280, Minibatch Loss= 1.810298, Training Accuracy= 0.41406\n", + "Iter 2560, Minibatch Loss= 1.653038, Training Accuracy= 0.48438\n", + "Iter 3840, Minibatch Loss= 1.088781, Training Accuracy= 0.65625\n", + "Iter 5120, Minibatch Loss= 0.936294, Training Accuracy= 0.67188\n", + "Iter 6400, Minibatch Loss= 0.814716, Training Accuracy= 0.75000\n", + "Iter 7680, Minibatch Loss= 0.643409, Training Accuracy= 0.78906\n", + "Iter 8960, Minibatch Loss= 0.671239, Training Accuracy= 0.78906\n", + "Iter 10240, Minibatch Loss= 0.474204, Training Accuracy= 0.83594\n", + "Iter 11520, Minibatch Loss= 0.427347, Training Accuracy= 0.86719\n", + "Iter 12800, Minibatch Loss= 0.424198, Training Accuracy= 0.85938\n", + "Iter 14080, Minibatch Loss= 0.458741, Training Accuracy= 0.84375\n", + "Iter 15360, Minibatch Loss= 0.467792, Training Accuracy= 0.86719\n", + "Iter 16640, Minibatch Loss= 0.199288, Training Accuracy= 0.94531\n", + "Iter 17920, Minibatch Loss= 0.339153, Training Accuracy= 0.89844\n", + "Iter 19200, Minibatch Loss= 0.307545, Training Accuracy= 0.92188\n", + "Iter 20480, Minibatch Loss= 0.240278, Training Accuracy= 0.90625\n", + "Iter 21760, Minibatch Loss= 0.253847, Training Accuracy= 0.92188\n", + "Iter 23040, Minibatch Loss= 0.202617, Training Accuracy= 0.92969\n", + "Iter 24320, Minibatch Loss= 0.290327, Training Accuracy= 0.91406\n", + "Iter 25600, Minibatch Loss= 0.168713, Training Accuracy= 0.96094\n", + "Iter 26880, Minibatch Loss= 0.156594, Training Accuracy= 0.96875\n", + "Iter 28160, Minibatch Loss= 0.333502, Training Accuracy= 0.88281\n", + "Iter 29440, Minibatch Loss= 0.246582, Training Accuracy= 0.91406\n", + "Iter 30720, Minibatch Loss= 0.231173, Training Accuracy= 0.92188\n", + "Iter 32000, Minibatch Loss= 0.218554, Training Accuracy= 0.91406\n", + "Iter 33280, Minibatch Loss= 0.154140, Training Accuracy= 0.93750\n", + "Iter 34560, Minibatch Loss= 0.202632, Training Accuracy= 0.93750\n", + "Iter 35840, Minibatch Loss= 0.205678, Training Accuracy= 0.92188\n", + "Iter 37120, Minibatch Loss= 0.274195, Training Accuracy= 0.92969\n", + "Iter 38400, Minibatch Loss= 0.236683, Training Accuracy= 0.94531\n", + "Iter 39680, Minibatch Loss= 0.132524, Training Accuracy= 0.96094\n", + "Iter 40960, Minibatch Loss= 0.144923, Training Accuracy= 0.92969\n", + "Iter 42240, Minibatch Loss= 0.102355, Training Accuracy= 0.96875\n", + "Iter 43520, Minibatch Loss= 0.231111, Training Accuracy= 0.94531\n", + "Iter 44800, Minibatch Loss= 0.205104, Training Accuracy= 0.94531\n", + "Iter 46080, Minibatch Loss= 0.166295, Training Accuracy= 0.95312\n", + "Iter 47360, Minibatch Loss= 0.143017, Training Accuracy= 0.96094\n", + "Iter 48640, Minibatch Loss= 0.126032, Training Accuracy= 0.95312\n", + "Iter 49920, Minibatch Loss= 0.226964, Training Accuracy= 0.92188\n", + "Iter 51200, Minibatch Loss= 0.093971, Training Accuracy= 0.96875\n", + "Iter 52480, Minibatch Loss= 0.074533, Training Accuracy= 0.96875\n", + "Iter 53760, Minibatch Loss= 0.099194, Training Accuracy= 0.97656\n", + "Iter 55040, Minibatch Loss= 0.083363, Training Accuracy= 0.96875\n", + "Iter 56320, Minibatch Loss= 0.099111, Training Accuracy= 0.96875\n", + "Iter 57600, Minibatch Loss= 0.172313, Training Accuracy= 0.94531\n", + "Iter 58880, Minibatch Loss= 0.067944, Training Accuracy= 0.98438\n", + "Iter 60160, Minibatch Loss= 0.076590, Training Accuracy= 0.98438\n", + "Iter 61440, Minibatch Loss= 0.122575, Training Accuracy= 0.94531\n", + "Iter 62720, Minibatch Loss= 0.129073, Training Accuracy= 0.96875\n", + "Iter 64000, Minibatch Loss= 0.074188, Training Accuracy= 0.97656\n", + "Iter 65280, Minibatch Loss= 0.079397, Training Accuracy= 0.96094\n", + "Iter 66560, Minibatch Loss= 0.120032, Training Accuracy= 0.96094\n", + "Iter 67840, Minibatch Loss= 0.161353, Training Accuracy= 0.94531\n", + "Iter 69120, Minibatch Loss= 0.124139, Training Accuracy= 0.96875\n", + "Iter 70400, Minibatch Loss= 0.153240, Training Accuracy= 0.95312\n", + "Iter 71680, Minibatch Loss= 0.091048, Training Accuracy= 0.97656\n", + "Iter 72960, Minibatch Loss= 0.106168, Training Accuracy= 0.97656\n", + "Iter 74240, Minibatch Loss= 0.089945, Training Accuracy= 0.96875\n", + "Iter 75520, Minibatch Loss= 0.103590, Training Accuracy= 0.97656\n", + "Iter 76800, Minibatch Loss= 0.037315, Training Accuracy= 0.99219\n", + "Iter 78080, Minibatch Loss= 0.161314, Training Accuracy= 0.96875\n", + "Iter 79360, Minibatch Loss= 0.079335, Training Accuracy= 0.99219\n", + "Iter 80640, Minibatch Loss= 0.054627, Training Accuracy= 0.97656\n", + "Iter 81920, Minibatch Loss= 0.071345, Training Accuracy= 0.97656\n", + "Iter 83200, Minibatch Loss= 0.086784, Training Accuracy= 0.96875\n", + "Iter 84480, Minibatch Loss= 0.047563, Training Accuracy= 0.98438\n", + "Iter 85760, Minibatch Loss= 0.103399, Training Accuracy= 0.98438\n", + "Iter 87040, Minibatch Loss= 0.062650, Training Accuracy= 0.98438\n", + "Iter 88320, Minibatch Loss= 0.024324, Training Accuracy= 0.99219\n", + "Iter 89600, Minibatch Loss= 0.064202, Training Accuracy= 0.98438\n", + "Iter 90880, Minibatch Loss= 0.036536, Training Accuracy= 1.00000\n", + "Iter 92160, Minibatch Loss= 0.025906, Training Accuracy= 0.99219\n", + "Iter 93440, Minibatch Loss= 0.060209, Training Accuracy= 0.97656\n", + "Iter 94720, Minibatch Loss= 0.154609, Training Accuracy= 0.96094\n", + "Iter 96000, Minibatch Loss= 0.068655, Training Accuracy= 0.98438\n", + "Iter 97280, Minibatch Loss= 0.053270, Training Accuracy= 0.98438\n", + "Iter 98560, Minibatch Loss= 0.074571, Training Accuracy= 0.98438\n", + "Iter 99840, Minibatch Loss= 0.065880, Training Accuracy= 0.98438\n", + "Optimization Finished!\n", + "Testing Accuracy: 0.984375\n" + ] + } + ], + "source": [ + "# Launch the graph\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + " step = 1\n", + " # Keep training until reach max iterations\n", + " while step * batch_size < training_iters:\n", + " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", + " # Reshape data to get 28 seq of 28 elements\n", + " batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n", + " # Run optimization op (backprop)\n", + " sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n", + " if step % display_step == 0:\n", + " # Calculate batch accuracy\n", + " acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n", + " # Calculate batch loss\n", + " loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n", + " print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n", + " \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n", + " \"{:.5f}\".format(acc)\n", + " step += 1\n", + " print \"Optimization Finished!\"\n", + "\n", + " # Calculate accuracy for 128 mnist test images\n", + " test_len = 128\n", + " test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n", + " test_label = mnist.test.labels[:test_len]\n", + " print \"Testing Accuracy:\", \\\n", + " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb new file mode 100644 index 00000000..6dee5504 --- /dev/null +++ b/notebooks/3_NeuralNetworks/.ipynb_checkpoints/recurrent_network-checkpoint.ipynb @@ -0,0 +1,221 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "'''\n", + "A Reccurent Neural Network (LSTM) implementation example using TensorFlow library.\n", + "This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/)\n", + "Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf\n", + "\n", + "Author: Aymeric Damien\n", + "Project: https://github.com/aymericdamien/TensorFlow-Examples/\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting /tmp/data/train-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/train-labels-idx1-ubyte.gz\n", + "Extracting /tmp/data/t10k-images-idx3-ubyte.gz\n", + "Extracting /tmp/data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "from tensorflow.python.ops import rnn, rnn_cell\n", + "import numpy as np\n", + "\n", + "# Import MINST data\n", + "from tensorflow.examples.tutorials.mnist import input_data\n", + "mnist = input_data.read_data_sets(\"/tmp/data/\", one_hot=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\nTo classify images using a reccurent neural network, we consider every image\\nrow as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\\nhandle 28 sequences of 28 steps for every sample.\\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "'''\n", + "To classify images using a reccurent neural network, we consider every image\n", + "row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\n", + "handle 28 sequences of 28 steps for every sample.\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "# Parameters\n", + "learning_rate = 0.001\n", + "training_iters = 100000\n", + "batch_size = 128\n", + "display_step = 10\n", + "\n", + "# Network Parameters\n", + "n_input = 28 # MNIST data input (img shape: 28*28)\n", + "n_steps = 28 # timesteps\n", + "n_hidden = 128 # hidden layer num of features\n", + "n_classes = 10 # MNIST total classes (0-9 digits)\n", + "\n", + "# tf Graph input\n", + "x = tf.placeholder(\"float\", [None, n_steps, n_input])\n", + "y = tf.placeholder(\"float\", [None, n_classes])\n", + "\n", + "# Define weights\n", + "weights = {\n", + " 'out': tf.Variable(tf.random_normal([n_hidden, n_classes]))\n", + "}\n", + "biases = {\n", + " 'out': tf.Variable(tf.random_normal([n_classes]))\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def RNN(x, weights, biases):\n", + "\n", + " # Prepare data shape to match `rnn` function requirements\n", + " # Current data input shape: (batch_size, n_steps, n_input)\n", + " # Required shape: 'n_steps' tensors list of shape (batch_size, n_input)\n", + " \n", + " # Permuting batch_size and n_steps\n", + " x = tf.transpose(x, [1, 0, 2])\n", + " # Reshaping to (n_steps*batch_size, n_input)\n", + " x = tf.reshape(x, [-1, n_input])\n", + " # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)\n", + " x = tf.split(0, n_steps, x)\n", + "\n", + " # Define a lstm cell with tensorflow\n", + " lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)\n", + "\n", + " # Get lstm cell output\n", + " outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n", + "\n", + " # Linear activation, using rnn inner loop last output\n", + " return tf.matmul(outputs[-1], weights['out']) + biases['out']\n", + "\n", + "pred = RNN(x, weights, biases)\n", + "\n", + "# Define loss and optimizer\n", + "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))\n", + "optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)\n", + "\n", + "# Evaluate model\n", + "correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))\n", + "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", + "\n", + "# Initializing the variables\n", + "init = tf." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "# Launch the graph\n", + "with tf.Session() as sess:\n", + " sess.run(init)\n", + " step = 1\n", + " # Keep training until reach max iterations\n", + " while step * batch_size < training_iters:\n", + " batch_x, batch_y = mnist.train.next_batch(batch_size)\n", + " # Reshape data to get 28 seq of 28 elements\n", + " batch_x = batch_x.reshape((batch_size, n_steps, n_input))\n", + " # Run optimization op (backprop)\n", + " sess.run(optimizer, feed_dict={x: batch_x, y: batch_y})\n", + " if step % display_step == 0:\n", + " # Calculate batch accuracy\n", + " acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})\n", + " # Calculate batch loss\n", + " loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})\n", + " print \"Iter \" + str(step*batch_size) + \", Minibatch Loss= \" + \\\n", + " \"{:.6f}\".format(loss) + \", Training Accuracy= \" + \\\n", + " \"{:.5f}\".format(acc)\n", + " step += 1\n", + " print \"Optimization Finished!\"\n", + "\n", + " # Calculate accuracy for 128 mnist test images\n", + " test_len = 128\n", + " test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))\n", + " test_label = mnist.test.labels[:test_len]\n", + " print \"Testing Accuracy:\", \\\n", + " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb b/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb index 581d50e8..ddce4e38 100644 --- a/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb +++ b/notebooks/3_NeuralNetworks/bidirectional_rnn.ipynb @@ -3,7 +3,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "'''\n", @@ -19,7 +21,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -34,7 +38,7 @@ ], "source": [ "import tensorflow as tf\n", - "from tensorflow.models.rnn import rnn, rnn_cell\n", + "from tensorflow.python.ops import rnn, rnn_cell\n", "import numpy as np\n", "\n", "# Import MINST data\n", @@ -45,7 +49,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "'''\n", @@ -57,8 +63,10 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "execution_count": 4, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "# Parameters\n", @@ -89,9 +97,34 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBiRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;31m# Define loss and optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mBiRNN\u001b[0;34m(x, weights, biases)\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# Old TensorFlow version only returns outputs not states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,\n\u001b[0;32m---> 26\u001b[0;31m dtype=tf.float32)\n\u001b[0m\u001b[1;32m 27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0;31m# Linear activation, using rnn inner loop last output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mbidirectional_rnn\u001b[0;34m(cell_fw, cell_bw, inputs, initial_state_fw, initial_state_bw, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 536\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"FW\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfw_scope\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 537\u001b[0m output_fw, output_state_fw = rnn(cell_fw, inputs, initial_state_fw, dtype,\n\u001b[0;32m--> 538\u001b[0;31m sequence_length, scope=fw_scope)\n\u001b[0m\u001b[1;32m 539\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 540\u001b[0m \u001b[0;31m# Backward direction\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mrnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 224\u001b[0m state_size=cell.state_size)\n\u001b[1;32m 225\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mconcat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# i = input_gate, j = new_input, f = forget_gate, o = output_gate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m_linear\u001b[0;34m(args, output_size, bias, bias_start, scope)\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"Linear\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 904\u001b[0m matrix = vs.get_variable(\n\u001b[0;32m--> 905\u001b[0;31m \"Matrix\", [total_arg_size, output_size], dtype=dtype)\n\u001b[0m\u001b[1;32m 906\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 907\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 1022\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 848\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 852\u001b[0m def _get_partitioned_variable(self,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m def _get_partitioned_variable(\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 631\u001b[0m \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 632\u001b[0;31m name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m 633\u001b[0m \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 634\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Variable BiRNN/FW/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 23, in BiRNN\n dtype=tf.float32)\n File \"\", line 31, in \n pred = BiRNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], "source": [ "def BiRNN(x, weights, biases):\n", "\n", @@ -134,98 +167,25 @@ "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", "\n", "# Initializing the variables\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, + "execution_count": 2, + "metadata": { + "collapsed": false + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iter 1280, Minibatch Loss= 1.689740, Training Accuracy= 0.36719\n", - "Iter 2560, Minibatch Loss= 1.477009, Training Accuracy= 0.44531\n", - "Iter 3840, Minibatch Loss= 1.245874, Training Accuracy= 0.53125\n", - "Iter 5120, Minibatch Loss= 0.990923, Training Accuracy= 0.64062\n", - "Iter 6400, Minibatch Loss= 0.752950, Training Accuracy= 0.71875\n", - "Iter 7680, Minibatch Loss= 1.023025, Training Accuracy= 0.61719\n", - "Iter 8960, Minibatch Loss= 0.921414, Training Accuracy= 0.68750\n", - "Iter 10240, Minibatch Loss= 0.719829, Training Accuracy= 0.75000\n", - "Iter 11520, Minibatch Loss= 0.468657, Training Accuracy= 0.86719\n", - "Iter 12800, Minibatch Loss= 0.654315, Training Accuracy= 0.78125\n", - "Iter 14080, Minibatch Loss= 0.595391, Training Accuracy= 0.83594\n", - "Iter 15360, Minibatch Loss= 0.392862, Training Accuracy= 0.83594\n", - "Iter 16640, Minibatch Loss= 0.421122, Training Accuracy= 0.92188\n", - "Iter 17920, Minibatch Loss= 0.311471, Training Accuracy= 0.88281\n", - "Iter 19200, Minibatch Loss= 0.276949, Training Accuracy= 0.92188\n", - "Iter 20480, Minibatch Loss= 0.170499, Training Accuracy= 0.94531\n", - "Iter 21760, Minibatch Loss= 0.419481, Training Accuracy= 0.86719\n", - "Iter 23040, Minibatch Loss= 0.183765, Training Accuracy= 0.92188\n", - "Iter 24320, Minibatch Loss= 0.386232, Training Accuracy= 0.86719\n", - "Iter 25600, Minibatch Loss= 0.335571, Training Accuracy= 0.88281\n", - "Iter 26880, Minibatch Loss= 0.169092, Training Accuracy= 0.92969\n", - "Iter 28160, Minibatch Loss= 0.247623, Training Accuracy= 0.92969\n", - "Iter 29440, Minibatch Loss= 0.242989, Training Accuracy= 0.94531\n", - "Iter 30720, Minibatch Loss= 0.253811, Training Accuracy= 0.92188\n", - "Iter 32000, Minibatch Loss= 0.169660, Training Accuracy= 0.93750\n", - "Iter 33280, Minibatch Loss= 0.291349, Training Accuracy= 0.90625\n", - "Iter 34560, Minibatch Loss= 0.172026, Training Accuracy= 0.95312\n", - "Iter 35840, Minibatch Loss= 0.186019, Training Accuracy= 0.93750\n", - "Iter 37120, Minibatch Loss= 0.298480, Training Accuracy= 0.89062\n", - "Iter 38400, Minibatch Loss= 0.158750, Training Accuracy= 0.92188\n", - "Iter 39680, Minibatch Loss= 0.162706, Training Accuracy= 0.94531\n", - "Iter 40960, Minibatch Loss= 0.339814, Training Accuracy= 0.86719\n", - "Iter 42240, Minibatch Loss= 0.068817, Training Accuracy= 0.99219\n", - "Iter 43520, Minibatch Loss= 0.188742, Training Accuracy= 0.93750\n", - "Iter 44800, Minibatch Loss= 0.176708, Training Accuracy= 0.92969\n", - "Iter 46080, Minibatch Loss= 0.096726, Training Accuracy= 0.96875\n", - "Iter 47360, Minibatch Loss= 0.220973, Training Accuracy= 0.92969\n", - "Iter 48640, Minibatch Loss= 0.226749, Training Accuracy= 0.94531\n", - "Iter 49920, Minibatch Loss= 0.188906, Training Accuracy= 0.94531\n", - "Iter 51200, Minibatch Loss= 0.145194, Training Accuracy= 0.95312\n", - "Iter 52480, Minibatch Loss= 0.168948, Training Accuracy= 0.95312\n", - "Iter 53760, Minibatch Loss= 0.069116, Training Accuracy= 0.97656\n", - "Iter 55040, Minibatch Loss= 0.228721, Training Accuracy= 0.93750\n", - "Iter 56320, Minibatch Loss= 0.152915, Training Accuracy= 0.95312\n", - "Iter 57600, Minibatch Loss= 0.126974, Training Accuracy= 0.96875\n", - "Iter 58880, Minibatch Loss= 0.078870, Training Accuracy= 0.97656\n", - "Iter 60160, Minibatch Loss= 0.225498, Training Accuracy= 0.95312\n", - "Iter 61440, Minibatch Loss= 0.111760, Training Accuracy= 0.97656\n", - "Iter 62720, Minibatch Loss= 0.161434, Training Accuracy= 0.97656\n", - "Iter 64000, Minibatch Loss= 0.207190, Training Accuracy= 0.94531\n", - "Iter 65280, Minibatch Loss= 0.103831, Training Accuracy= 0.96094\n", - "Iter 66560, Minibatch Loss= 0.153846, Training Accuracy= 0.93750\n", - "Iter 67840, Minibatch Loss= 0.082717, Training Accuracy= 0.96875\n", - "Iter 69120, Minibatch Loss= 0.199301, Training Accuracy= 0.95312\n", - "Iter 70400, Minibatch Loss= 0.139725, Training Accuracy= 0.96875\n", - "Iter 71680, Minibatch Loss= 0.169596, Training Accuracy= 0.95312\n", - "Iter 72960, Minibatch Loss= 0.142444, Training Accuracy= 0.96094\n", - "Iter 74240, Minibatch Loss= 0.145822, Training Accuracy= 0.95312\n", - "Iter 75520, Minibatch Loss= 0.129086, Training Accuracy= 0.94531\n", - "Iter 76800, Minibatch Loss= 0.078082, Training Accuracy= 0.97656\n", - "Iter 78080, Minibatch Loss= 0.151803, Training Accuracy= 0.94531\n", - "Iter 79360, Minibatch Loss= 0.050142, Training Accuracy= 0.98438\n", - "Iter 80640, Minibatch Loss= 0.136788, Training Accuracy= 0.95312\n", - "Iter 81920, Minibatch Loss= 0.130100, Training Accuracy= 0.94531\n", - "Iter 83200, Minibatch Loss= 0.058298, Training Accuracy= 0.98438\n", - "Iter 84480, Minibatch Loss= 0.120124, Training Accuracy= 0.96094\n", - "Iter 85760, Minibatch Loss= 0.064916, Training Accuracy= 0.97656\n", - "Iter 87040, Minibatch Loss= 0.137179, Training Accuracy= 0.93750\n", - "Iter 88320, Minibatch Loss= 0.138268, Training Accuracy= 0.95312\n", - "Iter 89600, Minibatch Loss= 0.072827, Training Accuracy= 0.97656\n", - "Iter 90880, Minibatch Loss= 0.123839, Training Accuracy= 0.96875\n", - "Iter 92160, Minibatch Loss= 0.087194, Training Accuracy= 0.96875\n", - "Iter 93440, Minibatch Loss= 0.083489, Training Accuracy= 0.97656\n", - "Iter 94720, Minibatch Loss= 0.131827, Training Accuracy= 0.95312\n", - "Iter 96000, Minibatch Loss= 0.098764, Training Accuracy= 0.96875\n", - "Iter 97280, Minibatch Loss= 0.115553, Training Accuracy= 0.94531\n", - "Iter 98560, Minibatch Loss= 0.079704, Training Accuracy= 0.96875\n", - "Iter 99840, Minibatch Loss= 0.064562, Training Accuracy= 0.98438\n", - "Optimization Finished!\n", - "Testing Accuracy: 0.992188\n" + "ename": "NameError", + "evalue": "name 'init' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m# Launch the graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minit\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m# Keep training until reach max iterations\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'init' is not defined" ] } ], @@ -259,6 +219,15 @@ " print \"Testing Accuracy:\", \\\n", " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -270,14 +239,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2.0 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.11" + "version": "2.7.12" } }, "nbformat": 4, diff --git a/notebooks/3_NeuralNetworks/recurrent_network.ipynb b/notebooks/3_NeuralNetworks/recurrent_network.ipynb index 32f3a939..2ccada45 100644 --- a/notebooks/3_NeuralNetworks/recurrent_network.ipynb +++ b/notebooks/3_NeuralNetworks/recurrent_network.ipynb @@ -3,7 +3,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "'''\n", @@ -18,8 +20,10 @@ }, { "cell_type": "code", - "execution_count": 1, - "metadata": {}, + "execution_count": 8, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -44,9 +48,22 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\nTo classify images using a reccurent neural network, we consider every image\\nrow as a sequence of pixels. Because MNIST image shape is 28*28px, we will then\\nhandle 28 sequences of 28 steps for every sample.\\n'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "'''\n", "To classify images using a reccurent neural network, we consider every image\n", @@ -57,8 +74,10 @@ }, { "cell_type": "code", - "execution_count": 2, - "metadata": {}, + "execution_count": 9, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "# Parameters\n", @@ -88,9 +107,33 @@ }, { "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "ename": "ValueError", + "evalue": "Variable RNN/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 18, in RNN\n outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n File \"\", line 23, in \n pred = RNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 21\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'out'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRNN\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbiases\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;31m# Define loss and optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m\u001b[0m in \u001b[0;36mRNN\u001b[0;34m(x, weights, biases)\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0;31m# Get lstm cell output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstates\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrnn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlstm_cell\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0;31m# Linear activation, using rnn inner loop last output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36mrnn\u001b[0;34m(cell, inputs, initial_state, dtype, sequence_length, scope)\u001b[0m\n\u001b[1;32m 224\u001b[0m state_size=cell.state_size)\n\u001b[1;32m 225\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 226\u001b[0;31m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_cell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 227\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtime\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mvarscope\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreuse_variables\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 212\u001b[0m \u001b[0;31m# pylint: disable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 213\u001b[0;31m \u001b[0mcall_cell\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 214\u001b[0m \u001b[0;31m# pylint: enable=cell-var-from-loop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msequence_length\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, state, scope)\u001b[0m\n\u001b[1;32m 309\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 310\u001b[0m \u001b[0mc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marray_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 311\u001b[0;31m \u001b[0mconcat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_linear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_units\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 312\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 313\u001b[0m \u001b[0;31m# i = input_gate, j = new_input, f = forget_gate, o = output_gate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/rnn_cell.pyc\u001b[0m in \u001b[0;36m_linear\u001b[0;34m(args, output_size, bias, bias_start, scope)\u001b[0m\n\u001b[1;32m 903\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvariable_scope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscope\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m\"Linear\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 904\u001b[0m matrix = vs.get_variable(\n\u001b[0;32m--> 905\u001b[0;31m \"Matrix\", [total_arg_size, output_size], dtype=dtype)\n\u001b[0m\u001b[1;32m 906\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 907\u001b[0m \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmath_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 1022\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1023\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1024\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 1025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1026\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 848\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 849\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 850\u001b[0;31m custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m 851\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 852\u001b[0m def _get_partitioned_variable(self,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m 344\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 345\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 347\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 348\u001b[0m def _get_partitioned_variable(\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 330\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m 332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m 630\u001b[0m \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 631\u001b[0m \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 632\u001b[0;31m name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m 633\u001b[0m \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 634\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValueError\u001b[0m: Variable RNN/BasicLSTMCell/Linear/Matrix already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n File \"\", line 18, in RNN\n outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)\n File \"\", line 23, in \n pred = RNN(x, weights, biases)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n" + ] + } + ], "source": [ "def RNN(x, weights, biases):\n", "\n", @@ -125,98 +168,29 @@ "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n", "\n", "# Initializing the variables\n", - "init = tf.initialize_all_variables()" + "init = tf.global_variables_initializer()" ] }, { "cell_type": "code", - "execution_count": 4, - "metadata": {}, + "execution_count": 10, + "metadata": { + "collapsed": false + }, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "Iter 1280, Minibatch Loss= 1.538532, Training Accuracy= 0.49219\n", - "Iter 2560, Minibatch Loss= 1.462834, Training Accuracy= 0.50781\n", - "Iter 3840, Minibatch Loss= 1.048393, Training Accuracy= 0.66406\n", - "Iter 5120, Minibatch Loss= 0.889872, Training Accuracy= 0.71875\n", - "Iter 6400, Minibatch Loss= 0.681855, Training Accuracy= 0.76562\n", - "Iter 7680, Minibatch Loss= 0.987207, Training Accuracy= 0.69531\n", - "Iter 8960, Minibatch Loss= 0.759543, Training Accuracy= 0.71094\n", - "Iter 10240, Minibatch Loss= 0.557055, Training Accuracy= 0.80469\n", - "Iter 11520, Minibatch Loss= 0.371352, Training Accuracy= 0.89844\n", - "Iter 12800, Minibatch Loss= 0.661293, Training Accuracy= 0.80469\n", - "Iter 14080, Minibatch Loss= 0.474259, Training Accuracy= 0.86719\n", - "Iter 15360, Minibatch Loss= 0.328436, Training Accuracy= 0.88281\n", - "Iter 16640, Minibatch Loss= 0.348017, Training Accuracy= 0.93750\n", - "Iter 17920, Minibatch Loss= 0.340086, Training Accuracy= 0.88281\n", - "Iter 19200, Minibatch Loss= 0.261532, Training Accuracy= 0.89844\n", - "Iter 20480, Minibatch Loss= 0.161785, Training Accuracy= 0.94531\n", - "Iter 21760, Minibatch Loss= 0.419619, Training Accuracy= 0.83594\n", - "Iter 23040, Minibatch Loss= 0.120714, Training Accuracy= 0.95312\n", - "Iter 24320, Minibatch Loss= 0.339519, Training Accuracy= 0.89062\n", - "Iter 25600, Minibatch Loss= 0.405463, Training Accuracy= 0.88281\n", - "Iter 26880, Minibatch Loss= 0.172193, Training Accuracy= 0.95312\n", - "Iter 28160, Minibatch Loss= 0.256769, Training Accuracy= 0.91406\n", - "Iter 29440, Minibatch Loss= 0.247753, Training Accuracy= 0.91406\n", - "Iter 30720, Minibatch Loss= 0.230820, Training Accuracy= 0.91406\n", - "Iter 32000, Minibatch Loss= 0.216861, Training Accuracy= 0.93750\n", - "Iter 33280, Minibatch Loss= 0.236337, Training Accuracy= 0.89062\n", - "Iter 34560, Minibatch Loss= 0.252351, Training Accuracy= 0.93750\n", - "Iter 35840, Minibatch Loss= 0.180090, Training Accuracy= 0.92188\n", - "Iter 37120, Minibatch Loss= 0.304125, Training Accuracy= 0.91406\n", - "Iter 38400, Minibatch Loss= 0.114474, Training Accuracy= 0.96094\n", - "Iter 39680, Minibatch Loss= 0.158405, Training Accuracy= 0.96875\n", - "Iter 40960, Minibatch Loss= 0.285858, Training Accuracy= 0.92188\n", - "Iter 42240, Minibatch Loss= 0.134199, Training Accuracy= 0.96094\n", - "Iter 43520, Minibatch Loss= 0.235847, Training Accuracy= 0.92969\n", - "Iter 44800, Minibatch Loss= 0.155971, Training Accuracy= 0.94531\n", - "Iter 46080, Minibatch Loss= 0.061549, Training Accuracy= 0.99219\n", - "Iter 47360, Minibatch Loss= 0.232569, Training Accuracy= 0.94531\n", - "Iter 48640, Minibatch Loss= 0.270348, Training Accuracy= 0.91406\n", - "Iter 49920, Minibatch Loss= 0.202416, Training Accuracy= 0.92188\n", - "Iter 51200, Minibatch Loss= 0.113857, Training Accuracy= 0.96094\n", - "Iter 52480, Minibatch Loss= 0.137900, Training Accuracy= 0.94531\n", - "Iter 53760, Minibatch Loss= 0.052416, Training Accuracy= 0.98438\n", - "Iter 55040, Minibatch Loss= 0.312064, Training Accuracy= 0.91406\n", - "Iter 56320, Minibatch Loss= 0.144335, Training Accuracy= 0.93750\n", - "Iter 57600, Minibatch Loss= 0.114723, Training Accuracy= 0.96875\n", - "Iter 58880, Minibatch Loss= 0.193597, Training Accuracy= 0.96094\n", - "Iter 60160, Minibatch Loss= 0.110877, Training Accuracy= 0.95312\n", - "Iter 61440, Minibatch Loss= 0.119864, Training Accuracy= 0.96094\n", - "Iter 62720, Minibatch Loss= 0.118780, Training Accuracy= 0.94531\n", - "Iter 64000, Minibatch Loss= 0.082259, Training Accuracy= 0.97656\n", - "Iter 65280, Minibatch Loss= 0.087364, Training Accuracy= 0.97656\n", - "Iter 66560, Minibatch Loss= 0.207975, Training Accuracy= 0.92969\n", - "Iter 67840, Minibatch Loss= 0.120612, Training Accuracy= 0.96875\n", - "Iter 69120, Minibatch Loss= 0.070608, Training Accuracy= 0.96875\n", - "Iter 70400, Minibatch Loss= 0.100786, Training Accuracy= 0.96094\n", - "Iter 71680, Minibatch Loss= 0.114746, Training Accuracy= 0.94531\n", - "Iter 72960, Minibatch Loss= 0.083427, Training Accuracy= 0.96875\n", - "Iter 74240, Minibatch Loss= 0.089978, Training Accuracy= 0.96094\n", - "Iter 75520, Minibatch Loss= 0.195322, Training Accuracy= 0.94531\n", - "Iter 76800, Minibatch Loss= 0.161109, Training Accuracy= 0.96094\n", - "Iter 78080, Minibatch Loss= 0.169762, Training Accuracy= 0.94531\n", - "Iter 79360, Minibatch Loss= 0.054240, Training Accuracy= 0.98438\n", - "Iter 80640, Minibatch Loss= 0.160100, Training Accuracy= 0.95312\n", - "Iter 81920, Minibatch Loss= 0.110728, Training Accuracy= 0.96875\n", - "Iter 83200, Minibatch Loss= 0.054918, Training Accuracy= 0.98438\n", - "Iter 84480, Minibatch Loss= 0.104170, Training Accuracy= 0.96875\n", - "Iter 85760, Minibatch Loss= 0.071871, Training Accuracy= 0.97656\n", - "Iter 87040, Minibatch Loss= 0.170529, Training Accuracy= 0.96094\n", - "Iter 88320, Minibatch Loss= 0.087350, Training Accuracy= 0.96875\n", - "Iter 89600, Minibatch Loss= 0.079943, Training Accuracy= 0.96875\n", - "Iter 90880, Minibatch Loss= 0.128451, Training Accuracy= 0.92969\n", - "Iter 92160, Minibatch Loss= 0.046963, Training Accuracy= 0.98438\n", - "Iter 93440, Minibatch Loss= 0.162998, Training Accuracy= 0.96875\n", - "Iter 94720, Minibatch Loss= 0.122588, Training Accuracy= 0.96094\n", - "Iter 96000, Minibatch Loss= 0.073954, Training Accuracy= 0.97656\n", - "Iter 97280, Minibatch Loss= 0.130790, Training Accuracy= 0.96094\n", - "Iter 98560, Minibatch Loss= 0.067689, Training Accuracy= 0.97656\n", - "Iter 99840, Minibatch Loss= 0.186411, Training Accuracy= 0.92188\n", - "Optimization Finished!\n", - "Testing Accuracy: 0.976562\n" + "ename": "InvalidArgumentError", + "evalue": "You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\nCaused by op u'Placeholder', defined at:\n File \"/usr/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n \"__main__\", fname, loader, pkg_name)\n File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n exec code in run_globals\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py\", line 3, in \n app.launch_new_instance()\n File \"/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py\", line 474, in start\n ioloop.IOLoop.instance().start()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py\", line 160, in start\n super(ZMQIOLoop, self).start()\n File \"/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py\", line 887, in start\n handler_func(fd_obj, events)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 433, in _handle_events\n self._handle_recv()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 465, in _handle_recv\n self._run_callback(callback, msg)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 407, in _run_callback\n callback(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n handler(stream, idents, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n user_expressions, allow_stdin)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n if self.run_code(code, result):\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 14, in \n x = tf.placeholder(\"float\", [None, n_steps, n_input])\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py\", line 1587, in placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 2043, in _placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py\", line 759, in apply_op\n op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 2240, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 1128, in __init__\n self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mInvalidArgumentError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0mbatch_x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_input\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;31m# Run optimization op (backprop)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_y\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdisplay_step\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;31m# Calculate batch accuracy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 764\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 765\u001b[0m result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m run_metadata_ptr)\n\u001b[0m\u001b[1;32m 767\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 768\u001b[0m \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 962\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 963\u001b[0m results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m 965\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 966\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m 1012\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1013\u001b[0m return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m 1015\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1016\u001b[0m return self._do_call(_prun_fn, self._session, handle, feed_dict,\n", + "\u001b[0;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m 1032\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1033\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1034\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1036\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mInvalidArgumentError\u001b[0m: You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n\nCaused by op u'Placeholder', defined at:\n File \"/usr/lib/python2.7/runpy.py\", line 174, in _run_module_as_main\n \"__main__\", fname, loader, pkg_name)\n File \"/usr/lib/python2.7/runpy.py\", line 72, in _run_code\n exec code in run_globals\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py\", line 3, in \n app.launch_new_instance()\n File \"/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py\", line 658, in launch_instance\n app.start()\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py\", line 474, in start\n ioloop.IOLoop.instance().start()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py\", line 160, in start\n super(ZMQIOLoop, self).start()\n File \"/usr/local/lib/python2.7/dist-packages/tornado/ioloop.py\", line 887, in start\n handler_func(fd_obj, events)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 433, in _handle_events\n self._handle_recv()\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 465, in _handle_recv\n self._run_callback(callback, msg)\n File \"/usr/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py\", line 407, in _run_callback\n callback(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/tornado/stack_context.py\", line 275, in null_wrapper\n return fn(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n return self.dispatch_shell(stream, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n handler(stream, idents, msg)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n user_expressions, allow_stdin)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n res = shell.run_cell(code, store_history=store_history, silent=silent)\n File \"/usr/local/lib/python2.7/dist-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n interactivity=interactivity, compiler=compiler, result=result)\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n if self.run_code(code, result):\n File \"/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n exec(code_obj, self.user_global_ns, self.user_ns)\n File \"\", line 14, in \n x = tf.placeholder(\"float\", [None, n_steps, n_input])\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/array_ops.py\", line 1587, in placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_array_ops.py\", line 2043, in _placeholder\n name=name)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/op_def_library.py\", line 759, in apply_op\n op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 2240, in create_op\n original_op=self._default_original_op, op_def=op_def)\n File \"/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py\", line 1128, in __init__\n self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): You must feed a value for placeholder tensor 'Placeholder' with dtype float\n\t [[Node: Placeholder = Placeholder[dtype=DT_FLOAT, shape=[], _device=\"/job:localhost/replica:0/task:0/gpu:0\"]()]]\n" ] } ], @@ -250,6 +224,15 @@ " print \"Testing Accuracy:\", \\\n", " sess.run(accuracy, feed_dict={x: test_data, y: test_label})" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] } ], "metadata": { @@ -261,14 +244,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2.0 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.11" + "version": "2.7.12" } }, "nbformat": 4, From 9e5d8675eebe3c275bfa434a99f63366ceff540f Mon Sep 17 00:00:00 2001 From: ajeetksingh Date: Sat, 14 Jan 2017 10:24:13 +0530 Subject: [PATCH 2/2] minor changes and few addendum files --- examples/3_NeuralNetworks/autoencoder.py | 2 +- .../bidirectional_multilayer_rnn.py | 2 +- .../3_NeuralNetworks/bidirectional_rnn.py | 2 +- .../3_NeuralNetworks/convolutional_network.py | 2 +- examples/3_NeuralNetworks/dynamic_rnn.py | 2 +- .../3_NeuralNetworks/multilayer_perceptron.py | 2 +- .../recurrent_multilayer_network.py | 5 +- .../recurrent_multilayernetwork.py | 114 ------------------ .../3_NeuralNetworks/recurrent_network.py | 2 +- 9 files changed, 10 insertions(+), 123 deletions(-) delete mode 100644 examples/3_NeuralNetworks/recurrent_multilayernetwork.py diff --git a/examples/3_NeuralNetworks/autoencoder.py b/examples/3_NeuralNetworks/autoencoder.py index f87f6b23..3a314e07 100644 --- a/examples/3_NeuralNetworks/autoencoder.py +++ b/examples/3_NeuralNetworks/autoencoder.py @@ -83,7 +83,7 @@ def decoder(x): optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(cost) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py b/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py index 25259769..c5e32e62 100644 --- a/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py +++ b/examples/3_NeuralNetworks/bidirectional_multilayer_rnn.py @@ -93,7 +93,7 @@ def BiRNN(x, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph t = time.time() diff --git a/examples/3_NeuralNetworks/bidirectional_rnn.py b/examples/3_NeuralNetworks/bidirectional_rnn.py index cadcdfd2..4e79153b 100644 --- a/examples/3_NeuralNetworks/bidirectional_rnn.py +++ b/examples/3_NeuralNetworks/bidirectional_rnn.py @@ -90,7 +90,7 @@ def BiRNN(x, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph t = time.time() diff --git a/examples/3_NeuralNetworks/convolutional_network.py b/examples/3_NeuralNetworks/convolutional_network.py index 81461237..1f10886f 100644 --- a/examples/3_NeuralNetworks/convolutional_network.py +++ b/examples/3_NeuralNetworks/convolutional_network.py @@ -104,7 +104,7 @@ def conv_net(x, weights, biases, dropout): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/dynamic_rnn.py b/examples/3_NeuralNetworks/dynamic_rnn.py index 9b9443d9..11b952d7 100644 --- a/examples/3_NeuralNetworks/dynamic_rnn.py +++ b/examples/3_NeuralNetworks/dynamic_rnn.py @@ -162,7 +162,7 @@ def dynamicRNN(x, seqlen, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/multilayer_perceptron.py b/examples/3_NeuralNetworks/multilayer_perceptron.py index b5c990f3..9086018a 100644 --- a/examples/3_NeuralNetworks/multilayer_perceptron.py +++ b/examples/3_NeuralNetworks/multilayer_perceptron.py @@ -64,7 +64,7 @@ def multilayer_perceptron(x, weights, biases): optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/recurrent_multilayer_network.py b/examples/3_NeuralNetworks/recurrent_multilayer_network.py index 8cb0d8e7..8e378e92 100644 --- a/examples/3_NeuralNetworks/recurrent_multilayer_network.py +++ b/examples/3_NeuralNetworks/recurrent_multilayer_network.py @@ -33,7 +33,7 @@ n_steps = 28 # timesteps n_hidden = 128 # hidden layer num of features n_classes = 10 # MNIST total classes (0-9 digits) -n_layers = 2 # Number of hidden layers +n_layers = 2 # Number of Hidden Layers # tf Graph input x = tf.placeholder("float", [None, n_steps, n_input]) @@ -63,6 +63,7 @@ def RNN(x, weights, biases): # Define a lstm cell with tensorflow lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) + # Define a multilayer lstm cell lstm_cell = rnn_cell.MultiRNNCell([lstm_cell] * n_layers, state_is_tuple=True) # Get lstm cell output @@ -82,7 +83,7 @@ def RNN(x, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: diff --git a/examples/3_NeuralNetworks/recurrent_multilayernetwork.py b/examples/3_NeuralNetworks/recurrent_multilayernetwork.py deleted file mode 100644 index 8cb0d8e7..00000000 --- a/examples/3_NeuralNetworks/recurrent_multilayernetwork.py +++ /dev/null @@ -1,114 +0,0 @@ -''' -A Recurrent Neural Network (LSTM) implementation example using TensorFlow library. -This example is using the MNIST database of handwritten digits (http://yann.lecun.com/exdb/mnist/) -Long Short Term Memory paper: http://deeplearning.cs.cmu.edu/pdfs/Hochreiter97_lstm.pdf - -Author: Aymeric Damien -Project: https://github.com/aymericdamien/TensorFlow-Examples/ -''' - -from __future__ import print_function - -import tensorflow as tf -from tensorflow.python.ops import rnn, rnn_cell - -# Import MNIST data -from tensorflow.examples.tutorials.mnist import input_data -mnist = input_data.read_data_sets("/tmp/data/", one_hot=True) - -''' -To classify images using a recurrent neural network, we consider every image -row as a sequence of pixels. Because MNIST image shape is 28*28px, we will then -handle 28 sequences of 28 steps for every sample. -''' - -# Parameters -learning_rate = 0.001 -training_iters = 100000 -batch_size = 128 -display_step = 10 - -# Network Parameters -n_input = 28 # MNIST data input (img shape: 28*28) -n_steps = 28 # timesteps -n_hidden = 128 # hidden layer num of features -n_classes = 10 # MNIST total classes (0-9 digits) -n_layers = 2 # Number of hidden layers - -# tf Graph input -x = tf.placeholder("float", [None, n_steps, n_input]) -y = tf.placeholder("float", [None, n_classes]) - -# Define weights -weights = { - 'out': tf.Variable(tf.random_normal([n_hidden, n_classes])) -} -biases = { - 'out': tf.Variable(tf.random_normal([n_classes])) -} - - -def RNN(x, weights, biases): - - # Prepare data shape to match `rnn` function requirements - # Current data input shape: (batch_size, n_steps, n_input) - # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) - - # Permuting batch_size and n_steps - x = tf.transpose(x, [1, 0, 2]) - # Reshaping to (n_steps*batch_size, n_input) - x = tf.reshape(x, [-1, n_input]) - # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) - x = tf.split(0, n_steps, x) - - # Define a lstm cell with tensorflow - lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) - lstm_cell = rnn_cell.MultiRNNCell([lstm_cell] * n_layers, state_is_tuple=True) - - # Get lstm cell output - outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32) - - # Linear activation, using rnn inner loop last output - return tf.matmul(outputs[-1], weights['out']) + biases['out'] - -pred = RNN(x, weights, biases) - -# Define loss and optimizer -cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) -optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost) - -# Evaluate model -correct_pred = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) -accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) - -# Initializing the variables -init = tf.initialize_all_variables() - -# Launch the graph -with tf.Session() as sess: - sess.run(init) - step = 1 - # Keep training until reach max iterations - while step * batch_size < training_iters: - batch_x, batch_y = mnist.train.next_batch(batch_size) - # Reshape data to get 28 seq of 28 elements - batch_x = batch_x.reshape((batch_size, n_steps, n_input)) - # Run optimization op (backprop) - sess.run(optimizer, feed_dict={x: batch_x, y: batch_y}) - if step % display_step == 0: - # Calculate batch accuracy - acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}) - # Calculate batch loss - loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y}) - print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \ - "{:.6f}".format(loss) + ", Training Accuracy= " + \ - "{:.5f}".format(acc)) - step += 1 - print("Optimization Finished!") - - # Calculate accuracy for 128 mnist test images - test_len = 128 - test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input)) - test_label = mnist.test.labels[:test_len] - print("Testing Accuracy:", \ - sess.run(accuracy, feed_dict={x: test_data, y: test_label})) diff --git a/examples/3_NeuralNetworks/recurrent_network.py b/examples/3_NeuralNetworks/recurrent_network.py index 21744364..9182f2bb 100644 --- a/examples/3_NeuralNetworks/recurrent_network.py +++ b/examples/3_NeuralNetworks/recurrent_network.py @@ -80,7 +80,7 @@ def RNN(x, weights, biases): accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Initializing the variables -init = tf.initialize_all_variables() +init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: