[MRG] Fix sklearn.metrics.classification._check_targets where y_true and y_pred are both binary but the union is multiclass (scikit-learn#8377)

* Fix _check_targets where y_true and y_pred are both binary

but the union of them is multiclass.

* Add entry in changelog

Author: lesteve

doc/whats_new.rst

@@ -260,6 +260,10 @@ Bug fixes
      obstructed pickling customizations of child-classes, when used in a
      multiple inheritance context.
      :issue:`8316` by :user:`Holger Peters <HolgerPeters>`.
+   - Fix a bug in :func:`sklearn.metrics.classification._check_targets`
+     which would return ``'binary'`` if ``y_true`` and ``y_pred`` were
+     both ``'binary'`` but the union of ``y_true`` and ``y_pred`` was
+     ``'multiclass'``. :issue:`8377` by `Loic Esteve`_.
 
    - Fix :func:`sklearn.linear_model.BayesianRidge.fit` to return 
      ridge parameter `alpha_` and `lambda_` consistent with calculated

sklearn/metrics/classification.py

@@ -91,6 +91,10 @@ def _check_targets(y_true, y_pred):
     if y_type in ["binary", "multiclass"]:
         y_true = column_or_1d(y_true)
         y_pred = column_or_1d(y_pred)
+        if y_type == "binary":
+            unique_values = np.union1d(y_true, y_pred)
+            if len(unique_values) > 2:
+                y_type = "multiclass"
 
     if y_type.startswith('multilabel'):
         y_true = csr_matrix(y_true)

sklearn/metrics/tests/test_classification.py

@@ -366,7 +366,8 @@ def test_matthews_corrcoef():
     y_true_inv = ["b" if i == "a" else "a" for i in y_true]
 
     assert_almost_equal(matthews_corrcoef(y_true, y_true_inv), -1)
-    y_true_inv2 = label_binarize(y_true, ["a", "b"]) * -1
+    y_true_inv2 = label_binarize(y_true, ["a", "b"])
+    y_true_inv2 = np.where(y_true_inv2, 'a', 'b')
     assert_almost_equal(matthews_corrcoef(y_true, y_true_inv2), -1)
 
     # For the zero vector case, the corrcoef cannot be calculated and should
@@ -379,8 +380,7 @@ def test_matthews_corrcoef():
 
     # And also for any other vector with 0 variance
     mcc = assert_warns_message(RuntimeWarning, 'invalid value encountered',
-                               matthews_corrcoef, y_true,
-                               rng.randint(-100, 100) * np.ones(20, dtype=int))
+                               matthews_corrcoef, y_true, ['a'] * len(y_true))
 
     # But will output 0
     assert_almost_equal(mcc, 0.)
@@ -1267,6 +1267,13 @@ def test__check_targets():
     assert_raise_message(ValueError, msg, _check_targets, y1, y2)
 
 
+def test__check_targets_multiclass_with_both_y_true_and_y_pred_binary():
+    # https://github.com/scikit-learn/scikit-learn/issues/8098
+    y_true = [0, 1]
+    y_pred = [0, -1]
+    assert_equal(_check_targets(y_true, y_pred)[0], 'multiclass')
+
+
 def test_hinge_loss_binary():
     y_true = np.array([-1, 1, 1, -1])
     pred_decision = np.array([-8.5, 0.5, 1.5, -0.3])