Added softmax exercise

Author: cxyzs7

.gitignore

@@ -52,3 +52,10 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 
+# PyCharm
+.idea/
+
+# Data and temporary data
+IMAGES.mat
+iterate.dat
+data/

softmax_exercise.py

@@ -0,0 +1,136 @@
+# CS294A/CS294W Softmax Exercise 
+
+from sklearn.datasets import fetch_mldata
+import numpy as np
+
+#  Instructions
+#  ------------
+# 
+#  This file contains code that helps you get started on the
+#  softmax exercise. You will need to write the softmax cost function 
+#  in softmaxCost.m and the softmax prediction function in softmaxPred.m. 
+#  For this exercise, you will not need to change any code in this file,
+#  or any other files other than those mentioned above.
+#  (However, you may be required to do so in later exercises)
+
+#======================================================================
+# STEP 0: Initialise constants and parameters
+#
+# Here we define and initialise some constants which allow your code
+# to be used more generally on any arbitrary input. 
+# We also initialise some parameters used for tuning the model.
+
+inputSize = 28 * 28     # Size of input vector (MNIST images are 28x28)
+numClasses = 10         # Number of classes (MNIST images fall into 10 classes)
+
+decayLambda = 1e-4      # Weight decay parameter
+
+#======================================================================
+# STEP 1: Load data
+#
+# In this section, we load the input and output data.
+# For softmax regression on MNIST pixels, 
+# the input data is the images, and 
+# the output data is the labels.
+#
+
+# Change the filenames if you've saved the files under different names
+# On some platforms, the files might be saved as 
+# train-images.idx3-ubyte / train-labels.idx1-ubyte
+
+mnist = fetch_mldata('MNIST original', data_home='./data/')
+
+# scale the pixel values to the range [0,1]
+images = np.float32(mnist.data)/255.0
+labels = mnist.target
+# Remap 0 to 10
+labels[labels == 0] = 10
+
+inputData = images
+
+# For debugging purposes, you may wish to reduce the size of the input data
+# in order to speed up gradient checking. 
+# Here, we create synthetic dataset using random data for testing
+
+DEBUG = True   # Set DEBUG to true when debugging.
+if DEBUG:
+    inputSize = 8
+    inputData = np.random.randn(100, 8)
+    labels = np.random.randint(10, size=(100, 1))
+
+# Randomly initialise theta
+theta = 0.005 * np.random.randn(numClasses * inputSize, 1)
+
+#======================================================================
+# STEP 2: Implement softmaxCost
+#
+# Implement softmaxCost in softmaxCost.m.
+
+#[cost, grad] = softmaxCost(theta, numClasses, inputSize, lambda, inputData, labels)
+                                     
+#======================================================================
+# STEP 3: Gradient checking
+#
+# As with any learning algorithm, you should always check that your
+# gradients are correct before learning the parameters.
+# 
+
+#if DEBUG:
+#    numGrad = computeNumericalGradient( @(x) softmaxCost(x, numClasses, ...
+#                                    inputSize, lambda, inputData, labels), theta)
+
+    # Use this to visually compare the gradients side by side
+#    disp([numGrad grad])
+
+    # Compare numerically computed gradients with those computed analytically
+#    diff = norm(numGrad-grad)/norm(numGrad+grad)
+#    disp(diff)
+    # The difference should be small. 
+    # In our implementation, these values are usually less than 1e-7.
+
+    # When your gradients are correct, congratulations!
+
+#======================================================================
+# STEP 4: Learning parameters
+#
+# Once you have verified that your gradients are correct,
+# you can start training your softmax regression code using softmaxTrain
+# (which uses minFunc).
+
+#options.maxIter = 100
+#softmaxModel = softmaxTrain(inputSize, numClasses, lambda, ...
+#                            inputData, labels, options)
+                          
+# Although we only use 100 iterations here to train a classifier for the 
+# MNIST data set, in practice, training for more iterations is usually
+# beneficial.
+
+#======================================================================
+# STEP 5: Testing
+#
+# You should now test your model against the test images.
+# To do this, you will first need to write softmaxPredict
+# (in softmaxPredict.m), which should return predictions
+# given a softmax model and the input data.
+
+#images = loadMNISTImages('mnist/t10k-images-idx3-ubyte')
+#labels = loadMNISTLabels('mnist/t10k-labels-idx1-ubyte')
+#labels(labels==0) = 10; # Remap 0 to 10
+
+#inputData = images
+
+# You will have to implement softmaxPredict in softmaxPredict.m
+#[pred] = softmaxPredict(softmaxModel, inputData)
+
+#acc = mean(labels(:) == pred(:))
+#fprintf('Accuracy: #0.3f##\n', acc * 100)
+
+# Accuracy is the proportion of correctly classified images
+# After 100 iterations, the results for our implementation were:
+#
+# Accuracy: 92.200#
+#
+# If your values are too low (accuracy less than 0.91), you should check 
+# your code for errors, and make sure you are training on the 
+# entire data set of 60000 28x28 training images 
+# (unless you modified the loading code, this should be the case)