Add sample_weight parameter to metrics.log_loss

  - Also modified binary output & multiclass tests in
    test_sample_weight_invariance to test for prediction
    inputs as probabilities
  - Updated What's New

Author: jatinshah

doc/whats_new.rst

@@ -44,10 +44,8 @@ Enhancements
      descent for :class:`linear_model.Lasso`, :class:`linear_model.ElasticNet`
      and related. By `Manoj Kumar`_.
 
-    - Add ``sample_weight`` parameter to `metrics.jaccard_similarity_score`.
-      By `Jatin Shah`.
-
-
+   - Add ``sample_weight`` parameter to `metrics.jaccard_similarity_score` and
+     `metrics.log_loss`. By `Jatin Shah`.
 
 Documentation improvements
 ..........................

sklearn/metrics/classification.py

@@ -92,6 +92,15 @@ def _check_clf_targets(y_true, y_pred):
     return y_type, y_true, y_pred
 
 
+def _weighted_sum(sample_score, sample_weight, normalize=False):
+    if normalize:
+        return np.average(sample_score, weights=sample_weight)
+    elif sample_weight is not None:
+        return np.dot(sample_score, sample_weight)
+    else:
+        return sample_score.sum()
+
+
 def accuracy_score(y_true, y_pred, normalize=True, sample_weight=None):
     """Accuracy classification score.
 
@@ -159,14 +168,7 @@ def accuracy_score(y_true, y_pred, normalize=True, sample_weight=None):
     else:
         score = y_true == y_pred
 
-    if normalize:
-        if sample_weight is not None:
-            return np.average(score, weights=sample_weight)
-        return np.mean(score)
-    else:
-        if sample_weight is not None:
-            return np.dot(score, sample_weight)
-        return np.sum(score)
+    return _weighted_sum(score, sample_weight, normalize)
 
 
 def confusion_matrix(y_true, y_pred, labels=None):
@@ -344,13 +346,7 @@ def jaccard_similarity_score(y_true, y_pred, normalize=True,
     else:
         score = y_true == y_pred
 
-    if normalize:
-        return np.average(score, weights=sample_weight)
-    else:
-        if sample_weight is not None:
-            return np.dot(score, sample_weight)
-        else:
-            return np.sum(score)
+    return _weighted_sum(score, sample_weight, normalize)
 
 
 def matthews_corrcoef(y_true, y_pred):
@@ -1317,7 +1313,7 @@ def hamming_loss(y_true, y_pred, classes=None):
         raise ValueError("{0} is not supported".format(y_type))
 
 
-def log_loss(y_true, y_pred, eps=1e-15, normalize=True):
+def log_loss(y_true, y_pred, eps=1e-15, normalize=True, sample_weight=None):
     """Log loss, aka logistic loss or cross-entropy loss.
 
     This is the loss function used in (multinomial) logistic regression
@@ -1345,6 +1341,9 @@ def log_loss(y_true, y_pred, eps=1e-15, normalize=True):
         If true, return the mean loss per sample.
         Otherwise, return the sum of the per-sample losses.
 
+    sample_weight : array-like of shape = [n_samples], optional
+        Sample weights.
+
     Returns
     -------
     loss : float
@@ -1393,8 +1392,9 @@ def log_loss(y_true, y_pred, eps=1e-15, normalize=True):
 
     # Renormalize
     Y /= Y.sum(axis=1)[:, np.newaxis]
-    loss = -(T * np.log(Y)).sum()
-    return loss / T.shape[0] if normalize else loss
+    loss = -(T * np.log(Y)).sum(axis=1)
+
+    return _weighted_sum(loss, sample_weight, normalize)
 
 
 def hinge_loss(y_true, pred_decision, pos_label=None, neg_label=None):

sklearn/metrics/tests/test_common.py

@@ -136,6 +136,8 @@
 
 THRESHOLDED_METRICS = {
     "log_loss": log_loss,
+    "unnormalized_log_loss": partial(log_loss, normalize=False),
+
     "hinge_loss": hinge_loss,
 
     "roc_auc_score": roc_auc_score,
@@ -239,6 +241,7 @@
 # Threshold-based metrics with "multilabel-indicator" format support
 THRESHOLDED_MULTILABEL_METRICS = [
     "log_loss",
+    "unnormalized_log_loss",
 
     "roc_auc_score", "weighted_roc_auc", "samples_roc_auc",
     "micro_roc_auc", "macro_roc_auc",
@@ -315,7 +318,6 @@
     "confusion_matrix",
     "hamming_loss",
     "hinge_loss",
-    "log_loss",
     "matthews_corrcoef_score",
 ]
 
@@ -532,7 +534,7 @@ def test_invariance_string_vs_numbers_labels():
                                        "invariance test".format(name))
 
     for name, metric in THRESHOLDED_METRICS.items():
-        if name in ("log_loss", "hinge_loss"):
+        if name in ("log_loss", "hinge_loss", "unnormalized_log_loss"):
             measure_with_number = metric(y1, y2)
             measure_with_str = metric(y1_str, y2)
             assert_array_equal(measure_with_number, measure_with_str,
@@ -968,23 +970,31 @@ def test_sample_weight_invariance(n_samples=50):
     random_state = check_random_state(0)
     y_true = random_state.randint(0, 2, size=(n_samples, ))
     y_pred = random_state.randint(0, 2, size=(n_samples, ))
+    y_score = random_state.random_sample(size=(n_samples,))
     for name in ALL_METRICS:
         if (name in METRICS_WITHOUT_SAMPLE_WEIGHT or
                 name in METRIC_UNDEFINED_MULTICLASS):
             continue
         metric = ALL_METRICS[name]
-        yield check_sample_weight_invariance, name, metric, y_true, y_pred
+        if name in THRESHOLDED_METRICS:
+            yield check_sample_weight_invariance, name, metric, y_true, y_score
+        else:
+            yield check_sample_weight_invariance, name, metric, y_true, y_pred
 
     # multiclass
     random_state = check_random_state(0)
     y_true = random_state.randint(0, 5, size=(n_samples, ))
     y_pred = random_state.randint(0, 5, size=(n_samples, ))
+    y_score = random_state.random_sample(size=(n_samples, 5))
     for name in ALL_METRICS:
         if (name in METRICS_WITHOUT_SAMPLE_WEIGHT or
                 name in METRIC_UNDEFINED_MULTICLASS):
             continue
         metric = ALL_METRICS[name]
-        yield check_sample_weight_invariance, name, metric, y_true, y_pred
+        if name in THRESHOLDED_METRICS:
+            yield check_sample_weight_invariance, name, metric, y_true, y_score
+        else:
+            yield check_sample_weight_invariance, name, metric, y_true, y_pred
 
     # multilabel sequence
     y_true = 2 * [(1, 2, ), (1, ), (0, ), (0, 1), (1, 2)]