FIX Infer pos_label in Display method from_cv_results (scikit-learn#32372)

Author: lucyleeow

doc/whats_new/upcoming_changes/sklearn.metrics/32372.fix.rst

@@ -0,0 +1,4 @@
+- :meth:`metrics.RocCurveDisplay.from_cv_results` will now infer `pos_label` as
+  `estimator.classes_[-1]`, using the estimator from `cv_results`, when
+  `pos_label=None`. Previously, an error was raised when `pos_label=None`.
+  By :user:`Lucy Liu <lucyleeow>`.

sklearn/metrics/_plot/roc_curve.py

@@ -665,8 +665,8 @@ def from_cv_results(
 
         pos_label : int, float, bool or str, default=None
             The class considered as the positive class when computing the ROC AUC
-            metrics. By default, `estimators.classes_[1]` is considered
-            as the positive class.
+            metrics. By default, `estimator.classes_[1]` (using `estimator` from
+            `cv_results`) is considered as the positive class.
 
         ax : matplotlib axes, default=None
             Axes object to plot on. If `None`, a new figure and axes is
@@ -730,24 +730,23 @@ def from_cv_results(
         <...>
         >>> plt.show()
         """
-        pos_label_ = cls._validate_from_cv_results_params(
+        cls._validate_from_cv_results_params(
             cv_results,
             X,
             y,
             sample_weight=sample_weight,
-            pos_label=pos_label,
         )
 
         fpr_folds, tpr_folds, auc_folds = [], [], []
         for estimator, test_indices in zip(
             cv_results["estimator"], cv_results["indices"]["test"]
         ):
             y_true = _safe_indexing(y, test_indices)
-            y_pred, _ = _get_response_values_binary(
+            y_pred, pos_label_ = _get_response_values_binary(
                 estimator,
                 _safe_indexing(X, test_indices),
                 response_method=response_method,
-                pos_label=pos_label_,
+                pos_label=pos_label,
             )
             sample_weight_fold = (
                 None

sklearn/metrics/_plot/tests/test_roc_curve_display.py

@@ -8,7 +8,7 @@
 from sklearn import clone
 from sklearn.compose import make_column_transformer
 from sklearn.datasets import load_breast_cancer, make_classification
-from sklearn.exceptions import NotFittedError
+from sklearn.exceptions import NotFittedError, UndefinedMetricWarning
 from sklearn.linear_model import LogisticRegression
 from sklearn.metrics import RocCurveDisplay, auc, roc_curve
 from sklearn.model_selection import cross_validate, train_test_split
@@ -264,7 +264,7 @@ def test_roc_curve_from_cv_results_param_validation(pyplot, data_binary):
 
     # `pos_label` inconsistency
     y_multi[y_multi == 1] = 2
-    with pytest.raises(ValueError, match=r"y takes value in \{0, 2\}"):
+    with pytest.warns(UndefinedMetricWarning, match="No positive samples in y_true"):
         RocCurveDisplay.from_cv_results(cv_results, X, y_multi)
 
     # `name` is list while `curve_kwargs` is None or dict
@@ -588,6 +588,18 @@ def test_roc_curve_from_cv_results_curve_kwargs(pyplot, data_binary, curve_kwarg
             assert color == curve_kwargs[idx]["c"]
 
 
+def test_roc_curve_from_cv_results_pos_label_inferred(pyplot, data_binary):
+    """Check `pos_label` inferred correctly by `from_cv_results(pos_label=None)`."""
+    X, y = data_binary
+    cv_results = cross_validate(
+        LogisticRegression(), X, y, cv=3, return_estimator=True, return_indices=True
+    )
+
+    disp = RocCurveDisplay.from_cv_results(cv_results, X, y, pos_label=None)
+    # Should be `estimator.classes_[1]`
+    assert disp.pos_label == 1
+
+
 def _check_chance_level(plot_chance_level, chance_level_kw, display):
     """Check chance level line and line styles correct."""
     import matplotlib as mpl

sklearn/utils/_plotting.py

@@ -77,7 +77,6 @@ def _validate_from_cv_results_params(
         y,
         *,
         sample_weight,
-        pos_label,
     ):
         check_matplotlib_support(f"{cls.__name__}.from_cv_results")
 
@@ -107,14 +106,6 @@ def _validate_from_cv_results_params(
             )
         check_consistent_length(X, y, sample_weight)
 
-        try:
-            pos_label = _check_pos_label_consistency(pos_label, y)
-        except ValueError as e:
-            # Adapt error message
-            raise ValueError(str(e).replace("y_true", "y"))
-
-        return pos_label
-
     @staticmethod
     def _get_legend_label(curve_legend_metric, curve_name, legend_metric_name):
         """Helper to get legend label using `name` and `legend_metric`"""

sklearn/utils/tests/test_plotting.py

@@ -128,7 +128,6 @@ def test_validate_from_predictions_params_returns(pyplot, name, pos_label, y_tru
                 "X": np.array([[1, 2], [3, 4]]),
                 "y": np.array([0, 1]),
                 "sample_weight": None,
-                "pos_label": None,
             },
             "`cv_results` does not contain one of the following",
         ),
@@ -142,7 +141,6 @@ def test_validate_from_predictions_params_returns(pyplot, name, pos_label, y_tru
                 "X": np.array([[1, 2]]),
                 "y": np.array([0, 1]),
                 "sample_weight": None,
-                "pos_label": None,
             },
             "`X` does not contain the correct number of",
         ),
@@ -156,7 +154,6 @@ def test_validate_from_predictions_params_returns(pyplot, name, pos_label, y_tru
                 # `y` not binary
                 "y": np.array([0, 2, 1, 3]),
                 "sample_weight": None,
-                "pos_label": None,
             },
             "The target `y` is not binary",
         ),
@@ -170,24 +167,9 @@ def test_validate_from_predictions_params_returns(pyplot, name, pos_label, y_tru
                 "y": np.array([0, 1, 0, 1]),
                 # `sample_weight` wrong length
                 "sample_weight": np.array([0.5]),
-                "pos_label": None,
             },
             "Found input variables with inconsistent",
         ),
-        (
-            {
-                "cv_results": {
-                    "estimator": "dummy",
-                    "indices": {"test": [[1, 2], [1, 2]], "train": [[3, 4], [3, 4]]},
-                },
-                "X": np.array([1, 2, 3, 4]),
-                "y": np.array([2, 3, 2, 3]),
-                "sample_weight": None,
-                # Not specified when `y` not in {0, 1} or {-1, 1}
-                "pos_label": None,
-            },
-            "y takes value in {2, 3} and pos_label is not specified",
-        ),
     ],
 )
 def test_validate_from_cv_results_params(pyplot, params, err_msg):