changed xfer_* to activation_*, modified docs and demo to match;

made internal regression use LinearRegression object

Author: dclambert

elm.py

@@ -9,8 +9,9 @@
 ELMRegressor, SimpleELMRegressor, SimpleELMClassifier).
 
 An Extreme Learning Machine (ELM) is a single layer feedforward
-network with a random hidden layer components and least-squares fitting
-of the hidden->output weights by default. [1][2]
+network with a random hidden layer components and ordinary linear
+least squares fitting of the hidden->output weights by default.
+[1][2]
 
 References
 ----------
@@ -23,12 +24,11 @@
 from abc import ABCMeta, abstractmethod
 
 import numpy as np
-from scipy.linalg import pinv2
 
 from sklearn.utils import as_float_array
-from sklearn.utils.extmath import safe_sparse_dot
 from sklearn.base import BaseEstimator, ClassifierMixin, RegressorMixin
 from sklearn.preprocessing import LabelBinarizer
+from sklearn.linear_model import LinearRegression
 
 from random_hidden_layer import SimpleRandomHiddenLayer
 
@@ -96,18 +96,19 @@ class ELMRegressor(BaseELM, RegressorMixin):
     ELMRegressor is a regressor based on the Extreme Learning Machine.
 
     An Extreme Learning Machine (ELM) is a single layer feedforward
-    network with a random hidden layer components and least-squares fitting
-    of the hidden->output weights by default. [1][2]
+    network with a random hidden layer components and ordinary linear
+    least squares fitting of the hidden->output weights by default.
+    [1][2]
 
     Parameters
     ----------
     `hidden_layer` : random_hidden_layer instance, optional
         (default=SimpleRandomHiddenLayer(random_state=0))
 
-    `regressor`    : linear_model instance, optional (default=None)
+    `regressor`    : regressor instance, optional (default=None)
         If provided, this object is used to perform the regression from hidden
         unit activations to the outputs and subsequent predictions.  If not
-        present, a simple least squares fit is performed internally.
+        present, an ordinary linear least squares fit is performed
 
     Attributes
     ----------
@@ -139,14 +140,19 @@ def __init__(self,
 
         super(ELMRegressor, self).__init__(hidden_layer, regressor)
 
-        self.coefs_ = None
         self.fitted_ = False
         self.hidden_activations_ = None
+        self._lin_reg = LinearRegression(copy_X=False,
+                                         normalize=False,
+                                         fit_intercept=False)
 
     def _fit_regression(self, y):
-        """fit regression using internal least squares/supplied regressor"""
+        """
+        fit regression using internal linear regression
+        or supplied regressor
+        """
         if (self.regressor is None):
-            self.coefs_ = safe_sparse_dot(pinv2(self.hidden_activations_), y)
+            self._lin_reg.fit(self.hidden_activations_, y)
         else:
             self.regressor.fit(self.hidden_activations_, y)
 
@@ -183,7 +189,7 @@ def fit(self, X, y):
     def _get_predictions(self, X):
         """get predictions using internal least squares/supplied regressor"""
         if (self.regressor is None):
-            preds = safe_sparse_dot(self.hidden_activations_, self.coefs_)
+            preds = self._lin_reg.predict(self.hidden_activations_)
         else:
             preds = self.regressor.predict(self.hidden_activations_)
 
@@ -219,18 +225,19 @@ class ELMClassifier(BaseELM, ClassifierMixin):
     ELMClassifier is a classifier based on the Extreme Learning Machine.
 
     An Extreme Learning Machine (ELM) is a single layer feedforward
-    network with a random hidden layer components and least-squares fitting
-    of the hidden->output weights by default. [1][2]
+    network with a random hidden layer components and ordinary linear
+    least squares fitting of the hidden->output weights by default.
+    [1][2]
 
     Parameters
     ----------
     `hidden_layer` : random_hidden_layer instance, optional
         (default=SimpleRandomHiddenLayer(random_state=0))
 
-    `regressor`    : linear_model instance, optional (default=None)
+    `regressor`    : regressor instance, optional (default=None)
         If provided, this object is used to perform the regression from hidden
         unit activations to the outputs and subsequent predictions.  If not
-        present, a simple least squares fit is performed internally.
+        present, an ordinary linear least squares fit is performed
 
     Attributes
     ----------
@@ -333,8 +340,9 @@ class SimpleELMRegressor(BaseEstimator, RegressorMixin):
     SimpleELMRegressor is a regressor based on the Extreme Learning Machine.
 
     An Extreme Learning Machine (ELM) is a single layer feedforward
-    network with a random hidden layer components and least-squares fitting
-    of the hidden->output weights by default. [1][2]
+    network with a random hidden layer components and ordinary linear
+    least squares fitting of the hidden->output weights by default.
+    [1][2]
 
     SimpleELMRegressor is a wrapper for an ELMRegressor that uses a
     SimpleRandomHiddenLayer and passes the __init__ parameters through
@@ -345,14 +353,14 @@ class SimpleELMRegressor(BaseEstimator, RegressorMixin):
     `n_hidden` : int, optional (default=20)
         Number of units to generate in the SimpleRandomHiddenLayer
 
-    `xfer_func` : {callable, string} optional (default='tanh')
+    `activation_func` : {callable, string} optional (default='tanh')
         Function used to transform input activation
         It must be one of 'tanh', 'sine', 'tribas', 'sigmoid', 'hardlim' or
         a callable.  If none is given, 'tanh' will be used. If a callable
         is given, it will be used to compute the hidden unit activations.
 
-    `xfer_args` : dictionary, optional (default=None)
-        Supplies keyword arguments for a callable xfer_func
+    `activation_args` : dictionary, optional (default=None)
+        Supplies keyword arguments for a callable activation_func
 
     `random_state`  : int, RandomState instance or None (default=None)
         Control the pseudo random number generator used to generate the
@@ -376,12 +384,13 @@ class SimpleELMRegressor(BaseEstimator, RegressorMixin):
               2006.
     """
 
-    def __init__(self, n_hidden=20, xfer_func='tanh', xfer_args=None,
+    def __init__(self, n_hidden=20,
+                 activation_func='tanh', activation_args=None,
                  random_state=None):
 
         self.n_hidden = n_hidden
-        self.xfer_func = xfer_func
-        self.xfer_args = xfer_args
+        self.activation_func = activation_func
+        self.activation_args = activation_args
         self.random_state = random_state
 
         self.elm_regressor_ = None
@@ -407,8 +416,8 @@ def fit(self, X, y):
             Returns an instance of self.
         """
         rhl = SimpleRandomHiddenLayer(n_hidden=self.n_hidden,
-                                      xfer_func=self.xfer_func,
-                                      xfer_args=self.xfer_args,
+                                      activation_func=self.activation_func,
+                                      activation_args=self.activation_args,
                                       random_state=self.random_state)
 
         self.elm_regressor_ = ELMRegressor(hidden_layer=rhl)
@@ -440,8 +449,9 @@ class SimpleELMClassifier(BaseEstimator, ClassifierMixin):
     SimpleELMClassifier is a classifier based on the Extreme Learning Machine.
 
     An Extreme Learning Machine (ELM) is a single layer feedforward
-    network with a random hidden layer components and least-squares fitting
-    of the hidden->output weights by default. [1][2]
+    network with a random hidden layer components and ordinary linear
+    least squares fitting of the hidden->output weights by default.
+    [1][2]
 
     SimpleELMClassifier is a wrapper for an ELMClassifier that uses a
     SimpleRandomHiddenLayer and passes the __init__ parameters through
@@ -452,14 +462,14 @@ class SimpleELMClassifier(BaseEstimator, ClassifierMixin):
     `n_hidden` : int, optional (default=20)
         Number of units to generate in the SimpleRandomHiddenLayer
 
-    `xfer_func` : {callable, string} optional (default='tanh')
+    `activation_func` : {callable, string} optional (default='tanh')
         Function used to transform input activation
         It must be one of 'tanh', 'sine', 'tribas', 'sigmoid', 'hardlim' or
         a callable.  If none is given, 'tanh' will be used. If a callable
         is given, it will be used to compute the hidden unit activations.
 
-    `xfer_args` : dictionary, optional (default=None)
-        Supplies keyword arguments for a callable xfer_func
+    `activation_args` : dictionary, optional (default=None)
+        Supplies keyword arguments for a callable activation_func
 
     `random_state`  : int, RandomState instance or None (default=None)
         Control the pseudo random number generator used to generate the
@@ -486,12 +496,13 @@ class SimpleELMClassifier(BaseEstimator, ClassifierMixin):
               2006.
     """
 
-    def __init__(self, n_hidden=20, xfer_func='tanh', xfer_args=None,
+    def __init__(self, n_hidden=20,
+                 activation_func='tanh', activation_args=None,
                  random_state=None):
 
         self.n_hidden = n_hidden
-        self.xfer_func = xfer_func
-        self.xfer_args = xfer_args
+        self.activation_func = activation_func
+        self.activation_args = activation_args
         self.random_state = random_state
 
         self.elm_classifier_ = None
@@ -538,8 +549,8 @@ def fit(self, X, y):
             Returns an instance of self.
         """
         rhl = SimpleRandomHiddenLayer(n_hidden=self.n_hidden,
-                                      xfer_func=self.xfer_func,
-                                      xfer_args=self.xfer_args,
+                                      activation_func=self.activation_func,
+                                      activation_args=self.activation_args,
                                       random_state=self.random_state)
 
         self.elm_classifier_ = ELMClassifier(hidden_layer=rhl)

elm_notebook.py

@@ -5,7 +5,8 @@
 
 # Demo python notebook for sklearn elm and random_hidden_layer modules
 #
-# Author: David C. Lambert
+# Author: David C. Lambert [dcl -at- panix -dot- com]
+# Copyright(c) 2013
 # License: Simple BSD
 
 # <codecell>
@@ -78,36 +79,36 @@ def res_dist(x, y, e, n_runs=100, random_state=None):
 # <codecell>
 
 # RBF tests
-elmc = ELMClassifier(RBFRandomHiddenLayer(xfer_func='gaussian'))
+elmc = ELMClassifier(RBFRandomHiddenLayer(activation_func='gaussian'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
-elmc = ELMClassifier(RBFRandomHiddenLayer(xfer_func='poly_spline', gamma=2))
+elmc = ELMClassifier(RBFRandomHiddenLayer(activation_func='poly_spline', gamma=2))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
-elmc = ELMClassifier(RBFRandomHiddenLayer(xfer_func='multiquadric'))
+elmc = ELMClassifier(RBFRandomHiddenLayer(activation_func='multiquadric'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
 # Simple tests
-elmc = ELMClassifier(SimpleRandomHiddenLayer(xfer_func='sine'))
+elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='sine'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
-elmc = ELMClassifier(SimpleRandomHiddenLayer(xfer_func='tanh'))
+elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='tanh'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
-elmc = ELMClassifier(SimpleRandomHiddenLayer(xfer_func='tribas'))
+elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='tribas'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
-elmc = ELMClassifier(SimpleRandomHiddenLayer(xfer_func='sigmoid'))
+elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='sigmoid'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
-elmc = ELMClassifier(SimpleRandomHiddenLayer(xfer_func='hardlim'))
+elmc = ELMClassifier(SimpleRandomHiddenLayer(activation_func='hardlim'))
 tr,ts = res_dist(irx, iry, elmc, n_runs=100, random_state=0)
 
 # <codecell>
 
 hardlim = (lambda a: np.array(a > 0.0, dtype=float))
 tribas = (lambda a: np.clip(1.0 - np.fabs(a), 0.0, 1.0))
-elmr = ELMRegressor(SimpleRandomHiddenLayer(random_state=0, xfer_func=tribas))
+elmr = ELMRegressor(SimpleRandomHiddenLayer(random_state=0, activation_func=tribas))
 elmr.fit(xtoy_train, ytoy_train)
 print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
 plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
@@ -131,8 +132,8 @@ def res_dist(x, y, e, n_runs=100, random_state=None):
 nh = 10
 (ctrs, _, _) = k_means(xtoy_train, nh)
 unit_rs = np.ones(nh)
-rhl = RBFRandomHiddenLayer(n_hidden=nh, xfer_func='poly_spline', gamma=5)
-#rhl = RBFRandomHiddenLayer(n_hidden=nh, xfer_func='multiquadric', gamma=1)
+rhl = RBFRandomHiddenLayer(n_hidden=nh, activation_func='poly_spline', gamma=3)
+#rhl = RBFRandomHiddenLayer(n_hidden=nh, activation_func='multiquadric', gamma=1)
 #rhl = RBFRandomHiddenLayer(n_hidden=nh, centers=ctrs, radii=unit_rs, gamma=4)
 elmr = ELMRegressor(hidden_layer=rhl)
 elmr.fit(xtoy_train, ytoy_train)
@@ -150,7 +151,7 @@ def powtanh_xfer(activations, power=1.0):
     return pow(np.tanh(activations), power)
 
 #tanh_rhl = SimpleRandomHiddenLayer(n_hidden=5000, random_state=0)
-tanh_rhl = SimpleRandomHiddenLayer(n_hidden=5000, xfer_func=powtanh_xfer, xfer_args={'power':2.0})
+tanh_rhl = SimpleRandomHiddenLayer(n_hidden=5000, activation_func=powtanh_xfer, activation_args={'power':2.0})
 elmc_tanh = ELMClassifier(hidden_layer=tanh_rhl)
 elmc_tanh.fit(dgx_train, dgy_train)
 print elmc_tanh.score(dgx_train, dgy_train), elmc_tanh.score(dgx_test, dgy_test)
@@ -191,7 +192,7 @@ def powtanh_xfer(activations, power=1.0):
 
 # <codecell>
 
-elmc = SimpleELMClassifier(n_hidden=500, xfer_func='hardlim')
+elmc = SimpleELMClassifier(n_hidden=500, activation_func='hardlim')
 elmc.fit(dgx_train, dgy_train)
 print elmc.score(dgx_train, dgy_train), elmc.score(dgx_test, dgy_test)
 
@@ -204,8 +205,11 @@ def powtanh_xfer(activations, power=1.0):
 
 # <codecell>
 
-elmr = SimpleELMRegressor(xfer_func='tribas')
+elmr = SimpleELMRegressor(activation_func='tribas')
 elmr.fit(xtoy_train, ytoy_train)
 print elmr.score(xtoy_train, ytoy_train), elmr.score(xtoy_test, ytoy_test)
 plot(xtoy, ytoy, xtoy, elmr.predict(xtoy))
 
+# <codecell>
+
+

plot_elm_comparison.py

@@ -137,17 +137,21 @@ def make_classifiers():
 
     # pass user defined transfer func
     sinsq = (lambda x: np.power(np.sin(x), 2.0))
-    srhl_sinsq = SimpleRandomHiddenLayer(n_hidden=nh, xfer_func=sinsq,
+    srhl_sinsq = SimpleRandomHiddenLayer(n_hidden=nh,
+                                         activation_func=sinsq,
                                          random_state=0)
 
     # use internal transfer funcs
-    srhl_tanh = SimpleRandomHiddenLayer(n_hidden=nh, xfer_func='tanh',
+    srhl_tanh = SimpleRandomHiddenLayer(n_hidden=nh,
+                                        activation_func='tanh',
                                         random_state=0)
 
-    srhl_tribas = SimpleRandomHiddenLayer(n_hidden=nh, xfer_func='tribas',
+    srhl_tribas = SimpleRandomHiddenLayer(n_hidden=nh,
+                                          activation_func='tribas',
                                           random_state=0)
 
-    srhl_hardlim = SimpleRandomHiddenLayer(n_hidden=nh, xfer_func='hardlim',
+    srhl_hardlim = SimpleRandomHiddenLayer(n_hidden=nh,
+                                           activation_func='hardlim',
                                            random_state=0)
 
     # use gaussian RBF