[MRG] BUG Clips log_loss calculation in neural_network (scikit-learn#16117)

Author: thomasjpfan

doc/whats_new/v0.23.rst

@@ -145,6 +145,13 @@ Changelog
   :pr:`16076` by :user:`Guillaume Lemaitre <glemaitre>` and  
   :user:`Alex Shacked <alexshacked>`.
 
+:mod:`sklearn.neural_network`
+.............................
+
+- |Fix| Increases the numerical stability of the logistic loss function in
+  :class:`neural_network.MLPClassifier` by clipping the probabilities.
+  :pr:`16117` by `Thomas Fan`_.
+
 :mod:`sklearn.preprocessing`
 ............................
 

sklearn/neural_network/_base.py

@@ -212,6 +212,8 @@ def log_loss(y_true, y_prob):
     loss : float
         The degree to which the samples are correctly predicted.
     """
+    eps = np.finfo(y_prob.dtype).eps
+    y_prob = np.clip(y_prob, eps, 1 - eps)
     if y_prob.shape[1] == 1:
         y_prob = np.append(1 - y_prob, y_prob, axis=1)
 
@@ -232,7 +234,7 @@ def binary_log_loss(y_true, y_prob):
     y_true : array-like or label indicator matrix
         Ground truth (correct) labels.
 
-    y_prob : array-like of float, shape = (n_samples, n_classes)
+    y_prob : array-like of float, shape = (n_samples, 1)
         Predicted probabilities, as returned by a classifier's
         predict_proba method.
 
@@ -241,6 +243,8 @@ def binary_log_loss(y_true, y_prob):
     loss : float
         The degree to which the samples are correctly predicted.
     """
+    eps = np.finfo(y_prob.dtype).eps
+    y_prob = np.clip(y_prob, eps, 1 - eps)
     return -(xlogy(y_true, y_prob) +
              xlogy(1 - y_true, 1 - y_prob)).sum() / y_prob.shape[0]
 

sklearn/neural_network/tests/test_base.py

@@ -0,0 +1,26 @@
+import pytest
+import numpy as np
+
+from sklearn.neural_network._base import binary_log_loss
+from sklearn.neural_network._base import log_loss
+
+
+def test_binary_log_loss_1_prob_finite():
+    # y_proba is equal to one should result in a finite logloss
+    y_true = np.array([[0, 0, 1]]).T
+    y_prob = np.array([[0.9, 1.0, 1.0]]).T
+
+    loss = binary_log_loss(y_true, y_prob)
+    assert np.isfinite(loss)
+
+
+@pytest.mark.parametrize("y_true, y_prob", [
+    (np.array([[1, 0, 0], [0, 1, 0]]),
+     np.array([[0., 1., 0.], [0.9, 0.05, 0.05]])),
+    (np.array([[0, 0, 1]]).T,
+     np.array([[0.9, 1.0, 1.0]]).T),
+])
+def test_log_loss_1_prob_finite(y_true, y_prob):
+    # y_proba is equal to 1 should result in a finite logloss
+    loss = log_loss(y_true, y_prob)
+    assert np.isfinite(loss)