Merge pull request scikit-learn#3401 from vene/scorer_weights

 [MRG] scorer: add sample_weight support (+test)

Author: arjoly

doc/whats_new.rst

@@ -26,8 +26,12 @@ Enhancements
 ............
 
 
+   - Add support for sample weights in scorer objects.  Metrics with sample
+     weight support will automatically benefit from it.
+
+
 Documentation improvements
-...........................
+..........................
 
 
 Bug fixes

sklearn/metrics/scorer.py

@@ -39,7 +39,7 @@ def __init__(self, score_func, sign, kwargs):
         self._sign = sign
 
     @abstractmethod
-    def __call__(self, estimator, X, y):
+    def __call__(self, estimator, X, y, sample_weight=None):
         pass
 
     def __repr__(self):
@@ -56,7 +56,7 @@ def _factory_args(self):
 
 
 class _PredictScorer(_BaseScorer):
-    def __call__(self, estimator, X, y_true):
+    def __call__(self, estimator, X, y_true, sample_weight=None):
         """Evaluate predicted target values for X relative to y_true.
 
         Parameters
@@ -71,17 +71,26 @@ def __call__(self, estimator, X, y_true):
         y_true : array-like
             Gold standard target values for X.
 
+        sample_weight : array-like, optional (default=None)
+            Sample weights.
+
         Returns
         -------
         score : float
             Score function applied to prediction of estimator on X.
         """
         y_pred = estimator.predict(X)
-        return self._sign * self._score_func(y_true, y_pred, **self._kwargs)
-
+        if sample_weight is not None:
+            return self._sign * self._score_func(y_true, y_pred,
+                                                 sample_weight=sample_weight,
+                                                 **self._kwargs)
+        else:
+            return self._sign * self._score_func(y_true, y_pred,
+                                                 **self._kwargs)
+                
 
 class _ProbaScorer(_BaseScorer):
-    def __call__(self, clf, X, y):
+    def __call__(self, clf, X, y, sample_weight=None):
         """Evaluate predicted probabilities for X relative to y_true.
 
         Parameters
@@ -97,20 +106,28 @@ def __call__(self, clf, X, y):
             Gold standard target values for X. These must be class labels,
             not probabilities.
 
+        sample_weight : array-like, optional (default=None)
+            Sample weights.
+
         Returns
         -------
         score : float
             Score function applied to prediction of estimator on X.
         """
         y_pred = clf.predict_proba(X)
-        return self._sign * self._score_func(y, y_pred, **self._kwargs)
+        if sample_weight is not None:
+            return self._sign * self._score_func(y, y_pred,
+                                                 sample_weight=sample_weight,
+                                                 **self._kwargs)
+        else:
+            return self._sign * self._score_func(y, y_pred, **self._kwargs)
 
     def _factory_args(self):
         return ", needs_proba=True"
 
 
 class _ThresholdScorer(_BaseScorer):
-    def __call__(self, clf, X, y):
+    def __call__(self, clf, X, y, sample_weight=None):
         """Evaluate decision function output for X relative to y_true.
 
         Parameters
@@ -128,6 +145,9 @@ def __call__(self, clf, X, y):
             Gold standard target values for X. These must be class labels,
             not decision function values.
 
+        sample_weight : array-like, optional (default=None)
+            Sample weights.
+
         Returns
         -------
         score : float
@@ -152,7 +172,12 @@ def __call__(self, clf, X, y):
             elif isinstance(y_pred, list):
                 y_pred = np.vstack([p[:, -1] for p in y_pred]).T
 
-        return self._sign * self._score_func(y, y_pred, **self._kwargs)
+        if sample_weight is not None:
+            return self._sign * self._score_func(y, y_pred,
+                                                 sample_weight=sample_weight,
+                                                 **self._kwargs)
+        else:
+            return self._sign * self._score_func(y, y_pred, **self._kwargs)
 
     def _factory_args(self):
         return ", needs_threshold=True"

sklearn/metrics/tests/test_score_objects.py

@@ -7,6 +7,8 @@
 from sklearn.utils.testing import assert_raises_regexp
 from sklearn.utils.testing import assert_true
 from sklearn.utils.testing import ignore_warnings
+from sklearn.utils.testing import assert_equal
+from sklearn.utils.testing import assert_not_equal
 
 from sklearn.metrics import (f1_score, r2_score, roc_auc_score, fbeta_score,
                              log_loss)
@@ -15,16 +17,25 @@
 from sklearn.metrics import make_scorer, SCORERS
 from sklearn.svm import LinearSVC
 from sklearn.cluster import KMeans
+from sklearn.dummy import DummyRegressor
 from sklearn.linear_model import Ridge, LogisticRegression
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.datasets import make_blobs
+from sklearn.datasets import make_classification
 from sklearn.datasets import make_multilabel_classification
 from sklearn.datasets import load_diabetes
 from sklearn.cross_validation import train_test_split, cross_val_score
 from sklearn.grid_search import GridSearchCV
 from sklearn.multiclass import OneVsRestClassifier
 
 
+REGRESSION_SCORERS = ['r2', 'mean_absolute_error', 'mean_squared_error']
+CLF_SCORERS = ['accuracy', 'f1', 'roc_auc', 'average_precision', 'precision',
+               'recall', 'log_loss',
+               'adjusted_rand_score'  # not really, but works
+               ]
+
+
 class EstimatorWithoutFit(object):
     """Dummy estimator to test check_scoring"""
     pass
@@ -229,3 +240,46 @@ def test_raises_on_score_list():
     grid_search = GridSearchCV(clf, scoring=f1_scorer_no_average,
                                param_grid={'max_depth': [1, 2]})
     assert_raises(ValueError, grid_search.fit, X, y)
+
+
+def test_scorer_sample_weight():
+    """Test that scorers support sample_weight or raise sensible errors"""
+
+    # Unlike the metrics invariance test, in the scorer case it's harder
+    # to ensure that, on the classifier output, weighted and unweighted
+    # scores really should be unequal.
+    X, y = make_classification(random_state=0)
+    X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
+    sample_weight = np.ones_like(y_test)
+    sample_weight[:10] = 0
+
+    # get sensible estimators for each metric
+    sensible_regr = DummyRegressor(strategy='median')
+    sensible_regr.fit(X_train, y_train)
+    sensible_clf = DecisionTreeClassifier()
+    sensible_clf.fit(X_train, y_train)
+    estimator = dict([(name, sensible_regr)
+                      for name in REGRESSION_SCORERS] +
+                     [(name, sensible_clf)
+                      for name in CLF_SCORERS])
+
+    for name, scorer in SCORERS.items():
+        try:
+            weighted = scorer(estimator[name], X_test, y_test,
+                              sample_weight=sample_weight)
+            ignored = scorer(estimator[name], X_test[10:], y_test[10:])
+            unweighted = scorer(estimator[name], X_test, y_test)
+            assert_not_equal(weighted, unweighted,
+                             "scorer {0} behaves identically when called with "
+                             "sample weights: {1} vs {2}".format(name,
+                                                                 weighted,
+                                                                 unweighted))
+            assert_equal(weighted, ignored,
+                         "scorer {0} behaves differently when ignoring "
+                         "samples and setting sample_weight to 0: "
+                         "{1} vs {2}".format(name, weighted, ignored))
+
+        except TypeError as e:
+            assert_true("sample_weight" in str(e),
+                        "scorer {0} raises unhelpful exception when called "
+                        "with sample weights: {1}".format(name, str(e)))