Add sigmoid derivative function

Author: lucasmoura

learning.py

@@ -3,7 +3,8 @@
 from utils import (
     removeall, unique, product, mode, argmax, argmax_random_tie, isclose,
     dotproduct, vector_add, scalar_vector_product, weighted_sample_with_replacement,
-    weighted_sampler, num_or_str, normalize, clip, sigmoid, print_table, DataFile
+    weighted_sampler, num_or_str, normalize, clip, sigmoid, print_table,
+    DataFile, sigmoid_derivative
 )
 
 import copy
@@ -541,6 +542,10 @@ def random_weights(min_value, max_value, num_weights):
     return [random.uniform(min_value, max_value) for i in range(num_weights)]
 
 
+def sigmoid_derivative_value(node):
+    return node.value * (1 - node.value)
+
+
 def BackPropagationLearner(dataset, net, learning_rate, epochs):
     """[Figure 18.23] The back-propagation algorithm for multilayer network"""
     # Initialise weights
@@ -558,7 +563,9 @@ def BackPropagationLearner(dataset, net, learning_rate, epochs):
     idx_t = [dataset.target]
     idx_i = dataset.inputs
     n_layers = len(net)
+    # output nodes
     o_nodes = net[-1]
+    # input nodes
     i_nodes = net[0]
 
     for epoch in range(epochs):
@@ -582,10 +589,10 @@ def BackPropagationLearner(dataset, net, learning_rate, epochs):
 
             # Compute outer layer delta
             o_units = len(o_nodes)
-            err = [t_val[i] - o_nodes[i].value
-                   for i in range(o_units)]
-            delta[-1] = [(o_nodes[i].value) * (1 - o_nodes[i].value) *
-                         (err[i]) for i in range(o_units)]
+            # Error for the MSE cost function
+            err = [t_val[i] - o_nodes[i].value for i in range(o_units)]
+            # The activation function used is the sigmoid function
+            delta[-1] = [sigmoid_derivative(o_nodes[i].value) * err[i] for i in range(o_units)]
 
             # Backward pass
             h_layers = n_layers - 2
@@ -594,11 +601,9 @@ def BackPropagationLearner(dataset, net, learning_rate, epochs):
                 h_units = len(layer)
                 nx_layer = net[i+1]
                 # weights from each ith layer node to each i + 1th layer node
-                w = [[node.weights[k] for node in nx_layer]
-                     for k in range(h_units)]
+                w = [[node.weights[k] for node in nx_layer] for k in range(h_units)]
 
-                delta[i] = [(layer[j].value) * (1 - layer[j].value) *
-                            dotproduct(w[j], delta[i+1])
+                delta[i] = [sigmoid_derivative(layer[j].value) * dotproduct(w[j], delta[i+1])
                             for j in range(h_units)]
 
             #  Update weights

tests/test_learning.py

@@ -99,3 +99,5 @@ def test_random_weights():
 
     for weight in test_weights:
         assert weight >= min_value and weight <= max_value
+
+

tests/test_utils.py

@@ -136,6 +136,16 @@ def test_sigmoid():
     assert isclose(0.2689414213699951, sigmoid(-1))
 
 
+def test_sigmoid_derivative():
+    value = 1
+
+    assert sigmoid_derivative(value) == 0
+
+    value = 3
+
+    assert sigmoid_derivative(value) == -6
+
+
 def test_step():
     assert step(1) == step(0.5) == 1
     assert step(0) == 1

utils.py

@@ -249,6 +249,10 @@ def clip(x, lowest, highest):
     return max(lowest, min(x, highest))
 
 
+def sigmoid_derivative(value):
+    return value * (1 - value)
+
+
 def sigmoid(x):
     """Return activation value of x with sigmoid function"""
     return 1/(1 + math.exp(-x))