ENH Add support for 'fit_params' to learning_curve (scikit-learn#18595)

Co-authored-by: Olivier Grisel <[email protected]>

Author: amy12xx

doc/whats_new/v0.24.rst

@@ -474,6 +474,10 @@ Changelog
   :pr:`18266` by :user:`Subrat Sahu <subrat93>`,
   :user:`Nirvan <Nirvan101>` and :user:`Arthur Book <ArthurBook>`.
 
+- |Enhancement| :func:`model_selection.learning_curve` now accept fit_params
+  to pass additional estimator parameters.
+  :pr:`18595` by :user:`Amanda Dsouza <amy12xx>`.
+
 :mod:`sklearn.multiclass`
 .........................
 

sklearn/model_selection/_validation.py

@@ -1214,8 +1214,8 @@ def learning_curve(estimator, X, y, *, groups=None,
                    train_sizes=np.linspace(0.1, 1.0, 5), cv=None,
                    scoring=None, exploit_incremental_learning=False,
                    n_jobs=None, pre_dispatch="all", verbose=0, shuffle=False,
-                   random_state=None, error_score=np.nan,
-                   return_times=False):
+                   random_state=None, error_score=np.nan, return_times=False,
+                   fit_params=None):
     """Learning curve.
 
     Determines cross-validated training and test scores for different training
@@ -1319,6 +1319,11 @@ def learning_curve(estimator, X, y, *, groups=None,
     return_times : bool, default=False
         Whether to return the fit and score times.
 
+    fit_params : dict, default=None
+        Parameters to pass to the fit method of the estimator.
+
+        .. versionadded:: 0.24
+
     Returns
     -------
     train_sizes_abs : array of shape (n_unique_ticks,)
@@ -1377,7 +1382,8 @@ def learning_curve(estimator, X, y, *, groups=None,
         classes = np.unique(y) if is_classifier(estimator) else None
         out = parallel(delayed(_incremental_fit_estimator)(
             clone(estimator), X, y, classes, train, test, train_sizes_abs,
-            scorer, verbose, return_times, error_score=error_score)
+            scorer, verbose, return_times, error_score=error_score,
+            fit_params=fit_params)
             for train, test in cv_iter
         )
         out = np.asarray(out).transpose((2, 1, 0))
@@ -1389,7 +1395,7 @@ def learning_curve(estimator, X, y, *, groups=None,
 
         results = parallel(delayed(_fit_and_score)(
             clone(estimator), X, y, scorer, train, test, verbose,
-            parameters=None, fit_params=None, return_train_score=True,
+            parameters=None, fit_params=fit_params, return_train_score=True,
             error_score=error_score, return_times=return_times)
             for train, test in train_test_proportions
         )
@@ -1472,10 +1478,12 @@ def _translate_train_sizes(train_sizes, n_max_training_samples):
 
 def _incremental_fit_estimator(estimator, X, y, classes, train, test,
                                train_sizes, scorer, verbose,
-                               return_times, error_score):
+                               return_times, error_score, fit_params):
     """Train estimator on training subsets incrementally and compute scores."""
     train_scores, test_scores, fit_times, score_times = [], [], [], []
     partitions = zip(train_sizes, np.split(train, train_sizes)[:-1])
+    if fit_params is None:
+        fit_params = {}
     for n_train_samples, partial_train in partitions:
         train_subset = train[:n_train_samples]
         X_train, y_train = _safe_split(estimator, X, y, train_subset)
@@ -1484,10 +1492,11 @@ def _incremental_fit_estimator(estimator, X, y, classes, train, test,
         X_test, y_test = _safe_split(estimator, X, y, test, train_subset)
         start_fit = time.time()
         if y_partial_train is None:
-            estimator.partial_fit(X_partial_train, classes=classes)
+            estimator.partial_fit(X_partial_train, classes=classes,
+                                  **fit_params)
         else:
             estimator.partial_fit(X_partial_train, y_partial_train,
-                                  classes=classes)
+                                  classes=classes, **fit_params)
         fit_time = time.time() - start_fit
         fit_times.append(fit_time)
 

sklearn/model_selection/tests/test_validation.py

@@ -25,6 +25,8 @@
 from sklearn.utils._testing import assert_allclose
 from sklearn.utils._mocking import CheckingClassifier, MockDataFrame
 
+from sklearn.utils.validation import _num_samples
+
 from sklearn.model_selection import cross_val_score, ShuffleSplit
 from sklearn.model_selection import cross_val_predict
 from sklearn.model_selection import cross_validate
@@ -114,9 +116,10 @@ def _is_training_data(self, X):
 
 class MockIncrementalImprovingEstimator(MockImprovingEstimator):
     """Dummy classifier that provides partial_fit"""
-    def __init__(self, n_max_train_sizes):
+    def __init__(self, n_max_train_sizes, expected_fit_params=None):
         super().__init__(n_max_train_sizes)
         self.x = None
+        self.expected_fit_params = expected_fit_params
 
     def _is_training_data(self, X):
         return self.x in X
@@ -125,6 +128,20 @@ def partial_fit(self, X, y=None, **params):
         self.train_sizes += X.shape[0]
         self.x = X[0]
 
+        if self.expected_fit_params:
+            missing = set(self.expected_fit_params) - set(params)
+            if missing:
+                raise AssertionError(
+                    f'Expected fit parameter(s) {list(missing)} not seen.'
+                )
+            for key, value in params.items():
+                if key in self.expected_fit_params and \
+                   _num_samples(value) != _num_samples(X):
+                    raise AssertionError(
+                        f'Fit parameter {key} has length {_num_samples(value)}'
+                        f'; expected {_num_samples(X)}.'
+                    )
+
 
 class MockEstimatorWithParameter(BaseEstimator):
     """Dummy classifier to test the validation curve"""
@@ -1249,6 +1266,48 @@ def test_learning_curve_with_shuffle():
                               test_scores_batch.mean(axis=1))
 
 
+def test_learning_curve_fit_params():
+    X = np.arange(100).reshape(10, 10)
+    y = np.array([0] * 5 + [1] * 5)
+    clf = CheckingClassifier(expected_fit_params=['sample_weight'])
+
+    err_msg = r"Expected fit parameter\(s\) \['sample_weight'\] not seen."
+    with pytest.raises(AssertionError, match=err_msg):
+        learning_curve(clf, X, y, error_score='raise')
+
+    err_msg = "Fit parameter sample_weight has length 1; expected"
+    with pytest.raises(AssertionError, match=err_msg):
+        learning_curve(clf, X, y, error_score='raise',
+                       fit_params={'sample_weight': np.ones(1)})
+    learning_curve(clf, X, y, error_score='raise',
+                   fit_params={'sample_weight': np.ones(10)})
+
+
+def test_learning_curve_incremental_learning_fit_params():
+    X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
+                               n_redundant=0, n_classes=2,
+                               n_clusters_per_class=1, random_state=0)
+    estimator = MockIncrementalImprovingEstimator(20, ['sample_weight'])
+    err_msg = r"Expected fit parameter\(s\) \['sample_weight'\] not seen."
+    with pytest.raises(AssertionError, match=err_msg):
+        learning_curve(estimator, X, y, cv=3,
+                       exploit_incremental_learning=True,
+                       train_sizes=np.linspace(0.1, 1.0, 10),
+                       error_score='raise')
+
+    err_msg = "Fit parameter sample_weight has length 3; expected"
+    with pytest.raises(AssertionError, match=err_msg):
+        learning_curve(estimator, X, y, cv=3,
+                       exploit_incremental_learning=True,
+                       train_sizes=np.linspace(0.1, 1.0, 10),
+                       error_score='raise',
+                       fit_params={'sample_weight': np.ones(3)})
+
+    learning_curve(estimator, X, y, cv=3, exploit_incremental_learning=True,
+                   train_sizes=np.linspace(0.1, 1.0, 10), error_score='raise',
+                   fit_params={'sample_weight': np.ones(2)})
+
+
 def test_validation_curve():
     X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
                                n_redundant=0, n_classes=2,