leriomaggio / deep-learning-keras-tensorflow

Code Issues 8 Pull requests 2 Actions Projects Wiki Security In

master

deep-learning-keras-tensorflow / 1. ANN / 1.1.1 Perceptron and Adaline.ipynb

Valerio Maggio
Repo Structure Refactoring + Merge from PyData London Version
History

0
contributors

871 lines (871 sloc)

145 KB
(exceprt from Python Machine Learning Essentials, Supplementary
Materials)

Sections
Implementing a perceptron learning algorithm in Python
Training a perceptron model on the Iris dataset
Adaptive linear neurons and the convergence of learning
Implementing an adaptive linear neuron in Python

In [1]: # Display plots in notebook

%matplotlib inline

# Define plot's default figure size

import matplotlib

Implementing a perceptron learning algorithm

in Python

[back to top]

In [2]: import numpy as np

class Perceptron(object):

"""Perceptron classifier.

Parameters

------------

eta : float

Learning rate (between 0.0 and 1.0)

n_iter : int

Passes over the training dataset.

Attributes

-----------

w_ : 1d-array

Weights after fitting.

errors_ : list

Number of misclassifications in every epoch.

"""

def __init__(self, eta=0.01, n_iter=10):

self.eta = eta

self.n_iter = n_iter

def fit(self, X, y):

"""Fit training data.

Parameters

----------

X : {array-like}, shape = [n_samples, n_feat

ures]

Training vectors, where n_samples is the

number of samples and

 n_features is the number of features.

y : array-like, shape = [n_samples]

Target values.

Returns

-------

self : object

"""

self.w_ = np.zeros(1 + X.shape[1])

self.errors_ = []

for _ in range(self.n_iter):

errors = 0

for xi, target in zip(X, y):

update = self.eta * (target - self.p
redict(xi))

self.w_[1:] += update * xi

self.w_[0] += update

errors += int(update != 0.0)

self.errors_.append(errors)

return self

def net_input(self, X):

"""Calculate net input"""

return np.dot(X, self.w_[1:]) + self.w_[0]

def predict(self, X):

"""Return class label after unit step"""

return np.where(self.net_input(X) >= 0.0, 1,

-1)

Training a perceptron model on the Iris dataset

[back to top]

Reading-in the Iris data

In [3]: import numpy as np

import pandas as pd

from sklearn.datasets import load_iris

iris = load_iris()

X = iris.data

y = iris.target

data = np.hstack((X, y[:, np.newaxis]))

labels = iris.target_names

features = iris.feature_names

df = pd.DataFrame(data, columns=iris.feature_names+[
'label'])

df.label = df.label.map({k:v for k,v in enumerate(la

bels)})

 df.tail()

Out[3]: sepal sepal petal petal

length width length width label
(cm) (cm) (cm) (cm)

145 6.7 3.0 5.2 2.3 virginica

146 6.3 2.5 5.0 1.9 virginica

147 6.5 3.0 5.2 2.0 virginica

148 6.2 3.4 5.4 2.3 virginica

149 5.9 3.0 5.1 1.8 virginica

Plotting the Iris data

In [4]: import numpy as np

import matplotlib.pyplot as plt

# select setosa and versicolor

y = df.iloc[0:100, 4].values

y = np.where(y == 'setosa', -1, 1)

# extract sepal length and petal length

X = df.iloc[0:100, [0, 2]].values

# plot data

plt.scatter(X[:50, 0], X[:50, 1],

color='red', marker='o', label='setosa')

plt.scatter(X[50:100, 0], X[50:100, 1],

color='blue', marker='x', label='versico

lor')

plt.xlabel('petal length [cm]')

plt.ylabel('sepal length [cm]')

plt.legend(loc='upper left')

plt.show()

Training the perceptron model

In [5]: ppn = Perceptron(eta=0.1, n_iter=10)

ppn.fit(X, y)

plt.plot(range(1, len(ppn.errors_) + 1), ppn.errors_

, marker='o')

plt.xlabel('Epochs')

plt.ylabel('Number of misclassifications')

plt.tight_layout()

plt.show()

A function for plotting decision regions

In [6]: from matplotlib.colors import ListedColormap

def plot_decision_regions(X, y, classifier, resoluti

on=0.02):

# setup marker generator and color map

markers = ('s', 'x', 'o', '^', 'v')

colors = ('red', 'blue', 'lightgreen', 'gray',

'cyan')

cmap = ListedColormap(colors[:len(np.unique(y
))])

# plot the decision surface

x1_min, x1_max = X[:, 0].min() - 1, X[:, 0].max

() + 1

x2_min, x2_max = X[:, 1].min() - 1, X[:, 1].max

 () + 1

xx1, xx2 = np.meshgrid(np.arange(x1_min, x1_max,

resolution),

np.arange(x2_min, x2_max, r
esolution))

Z = classifier.predict(np.array([xx1.ravel(), xx
2.ravel()]).T)

Z = Z.reshape(xx1.shape)

plt.contourf(xx1, xx2, Z, alpha=0.4, cmap=cmap)

plt.xlim(xx1.min(), xx1.max())

plt.ylim(xx2.min(), xx2.max())

# plot class samples

for idx, cl in enumerate(np.unique(y)):

plt.scatter(x=X[y == cl, 0], y=X[y == cl, 1

],

alpha=0.8, c=cmap(idx),

marker=markers[idx], label=cl)

In [7]: plot_decision_regions(X, y, classifier=ppn)

plt.xlabel('sepal length [cm]')

plt.ylabel('petal length [cm]')

plt.legend(loc='upper left')

plt.tight_layout()

plt.show()

Adaptive linear neurons and the convergence

of learning

[back to top]

Implementing an adaptive linear neuron in

Python

In [8]: class AdalineGD(object):

"""ADAptive LInear NEuron classifier.

Parameters

------------

eta : float

Learning rate (between 0.0 and 1.0)

n_iter : int

Passes over the training dataset.

Attributes

-----------

w_ : 1d-array

Weights after fitting.

errors_ : list

Number of misclassifications in every epoch.

"""

def __init__(self, eta=0.01, n_iter=50):

self.eta = eta

self.n_iter = n_iter

def fit(self, X, y):

""" Fit training data.

Parameters

----------

X : {array-like}, shape = [n_samples, n_feat

ures]

Training vectors, where n_samples is the

number of samples and

n_features is the number of features.

y : array-like, shape = [n_samples]

Target values.

Returns

-------

self : object

"""

self.w_ = np.zeros(1 + X.shape[1])

self.cost_ = []

for i in range(self.n_iter):

output = self.net_input(X)

errors = (y - output)

self.w_[1:] += self.eta * X.T.dot(errors

)

self.w_[0] += self.eta * errors.sum()

cost = (errors**2).sum() / 2.0

self.cost_.append(cost)

return self

def net_input(self, X):

"""Calculate net input"""

return np.dot(X, self.w_[1:]) + self.w_[0]

def activation(self, X):

"""Compute linear activation"""

return self.net_input(X)

def predict(self, X):

"""Return class label after unit step"""

return np.where(self.activation(X) >= 0.0, 1

 , -1)

In [9]: fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(8,

4))

ada1 = AdalineGD(n_iter=10, eta=0.01).fit(X, y)

ax[0].plot(range(1, len(ada1.cost_) + 1), np.log10(a

da1.cost_), marker='o')

ax[0].set_xlabel('Epochs')

ax[0].set_ylabel('log(Sum-squared-error)')

ax[0].set_title('Adaline - Learning rate 0.01')

ada2 = AdalineGD(n_iter=10, eta=0.0001).fit(X, y)

ax[1].plot(range(1, len(ada2.cost_) + 1), ada2.cost_

, marker='o')

ax[1].set_xlabel('Epochs')

ax[1].set_ylabel('Sum-squared-error')

ax[1].set_title('Adaline - Learning rate 0.0001')

plt.tight_layout()

plt.show()

Standardizing features and re-training adaline

In [10]: # standardize features

X_std = np.copy(X)

X_std[:,0] = (X[:,0] - X[:,0].mean()) / X[:,0].std()

X_std[:,1] = (X[:,1] - X[:,1].mean()) / X[:,1].std()

In [11]: ada = AdalineGD(n_iter=15, eta=0.01)

ada.fit(X_std, y)

plot_decision_regions(X_std, y, classifier=ada)

plt.title('Adaline - Gradient Descent')

plt.xlabel('sepal length [standardized]')

plt.ylabel('petal length [standardized]')

plt.legend(loc='upper left')

plt.tight_layout()

plt.show()

plt.plot(range(1, len(ada.cost_) + 1), ada.cost_, ma

rker='o')

plt.xlabel('Epochs')

 plt.ylabel('Sum-squared-error')

plt.tight_layout()

plt.show()

Large scale machine learning and stochastic gradient

descent

[back to top]

In [12]: from numpy.random import seed

class AdalineSGD(object):

"""ADAptive LInear NEuron classifier.

Parameters

------------

eta : float

Learning rate (between 0.0 and 1.0)

n_iter : int

Passes over the training dataset.

Attributes

 -----------

w_ : 1d-array

Weights after fitting.

errors_ : list

Number of misclassifications in every epoch.

shuffle : bool (default: True)

Shuffles training data every epoch if True t

o prevent cycles.

random_state : int (default: None)

Set random state for shuffling and initializ

ing the weights.

"""

def __init__(self, eta=0.01, n_iter=10, shuffle=

True, random_state=None):

self.eta = eta

self.n_iter = n_iter

self.w_initialized = False

self.shuffle = shuffle

if random_state:

seed(random_state)

def fit(self, X, y):

""" Fit training data.

Parameters

----------

X : {array-like}, shape = [n_samples, n_feat

ures]

Training vectors, where n_samples is the

number of samples and

n_features is the number of features.

y : array-like, shape = [n_samples]

Target values.

Returns

-------

self : object

"""

self._initialize_weights(X.shape[1])

self.cost_ = []

for i in range(self.n_iter):

if self.shuffle:

X, y = self._shuffle(X, y)

cost = []

for xi, target in zip(X, y):

cost.append(self._update_weights(xi,
target))

avg_cost = sum(cost)/len(y)

self.cost_.append(avg_cost)

return self

def partial_fit(self, X, y):

"""Fit training data without reinitializing

the weights"""

if not self.w_initialized:

self._initialize_weights(X.shape[1])

if y.ravel().shape[0] > 1:

for xi, target in zip(X, y):

self._update_weights(xi, target)

else:

 self._update_weights(X, y)

return self

def _shuffle(self, X, y):

"""Shuffle training data"""

r = np.random.permutation(len(y))

return X[r], y[r]

def _initialize_weights(self, m):

"""Initialize weights to zeros"""

self.w_ = np.zeros(1 + m)

self.w_initialized = True

def _update_weights(self, xi, target):

"""Apply Adaline learning rule to update the

weights"""

output = self.net_input(xi)

error = (target - output)

self.w_[1:] += self.eta * xi.dot(error)

self.w_[0] += self.eta * error

cost = 0.5 * error**2

return cost

def net_input(self, X):

"""Calculate net input"""

return np.dot(X, self.w_[1:]) + self.w_[0]

def activation(self, X):

"""Compute linear activation"""

return self.net_input(X)

def predict(self, X):

"""Return class label after unit step"""

return np.where(self.activation(X) >= 0.0, 1

, -1)

In [13]: ada = AdalineSGD(n_iter=15, eta=0.01, random_state=1

)

ada.fit(X_std, y)

plot_decision_regions(X_std, y, classifier=ada)

plt.title('Adaline - Stochastic Gradient Descent')

plt.xlabel('sepal length [standardized]')

plt.ylabel('petal length [standardized]')

plt.legend(loc='upper left')