[MRG+2] convert to boolean arrays for boolean distances (scikit-learn#6932)

for example jaccard.

Author: TomDLT

doc/whats_new.rst

@@ -130,10 +130,6 @@ Enhancements
      (`#4294 <https://github.com/scikit-learn/scikit-learn/pull/4294>`_) by
      `Raghav R V`_.
 
-   - The random forest, extra trees and decision tree estimators now has a
-     method ``decision_path`` which returns the decision path of samples in
-     the tree. By `Arnaud Joly`_.
-
    - The random forest, extra tree and decision tree estimators now has a
      method ``decision_path`` which returns the decision path of samples in
      the tree. By `Arnaud Joly`_.
@@ -283,6 +279,10 @@ Bug fixes
       with them as parameters, could not be passed to :func:`base.clone`.
       By `Loic Esteve`_.
 
+    - :func:`pairwise_distances` now converts arrays to boolean arrays when
+      required in scipy.spatial.distance.
+      (`#5460 https://github.com/scikit-learn/scikit-learn/pull/5460>`_)
+      By `Tom Dupre la Tour`_.
 
 API changes summary
 -------------------

sklearn/metrics/pairwise.py

@@ -54,7 +54,7 @@ def _return_float_dtype(X, Y):
     return X, Y, dtype
 
 
-def check_pairwise_arrays(X, Y, precomputed=False):
+def check_pairwise_arrays(X, Y, precomputed=False, dtype=None):
     """ Set X and Y appropriately and checks inputs
 
     If Y is None, it is set as a pointer to X (i.e. not a copy).
@@ -64,9 +64,9 @@ def check_pairwise_arrays(X, Y, precomputed=False):
 
     Specifically, this function first ensures that both X and Y are arrays,
     then checks that they are at least two dimensional while ensuring that
-    their elements are floats. Finally, the function checks that the size
-    of the second dimension of the two arrays is equal, or the equivalent
-    check for a precomputed distance matrix.
+    their elements are floats (or dtype if provided). Finally, the function
+    checks that the size of the second dimension of the two arrays is equal, or
+    the equivalent check for a precomputed distance matrix.
 
     Parameters
     ----------
@@ -78,6 +78,12 @@ def check_pairwise_arrays(X, Y, precomputed=False):
         True if X is to be treated as precomputed distances to the samples in
         Y.
 
+    dtype : string, type, list of types or None (default=None)
+        Data type required for X and Y. If None, the dtype will be an
+        appropriate float type selected by _return_float_dtype.
+
+        .. versionadded:: 0.18
+
     Returns
     -------
     safe_X : {array-like, sparse matrix}, shape (n_samples_a, n_features)
@@ -88,13 +94,21 @@ def check_pairwise_arrays(X, Y, precomputed=False):
         If Y was None, safe_Y will be a pointer to X.
 
     """
-    X, Y, dtype = _return_float_dtype(X, Y)
+    X, Y, dtype_float = _return_float_dtype(X, Y)
+
+    warn_on_dtype = dtype is not None
+    estimator = 'check_pairwise_arrays'
+    if dtype is None:
+        dtype = dtype_float
 
     if Y is X or Y is None:
-        X = Y = check_array(X, accept_sparse='csr', dtype=dtype)
+        X = Y = check_array(X, accept_sparse='csr', dtype=dtype,
+                            warn_on_dtype=warn_on_dtype, estimator=estimator)
     else:
-        X = check_array(X, accept_sparse='csr', dtype=dtype)
-        Y = check_array(Y, accept_sparse='csr', dtype=dtype)
+        X = check_array(X, accept_sparse='csr', dtype=dtype,
+                        warn_on_dtype=warn_on_dtype, estimator=estimator)
+        Y = check_array(Y, accept_sparse='csr', dtype=dtype,
+                        warn_on_dtype=warn_on_dtype, estimator=estimator)
 
     if precomputed:
         if X.shape[1] != Y.shape[0]:
@@ -1208,7 +1222,11 @@ def pairwise_distances(X, Y=None, metric="euclidean", n_jobs=1, **kwds):
         if issparse(X) or issparse(Y):
             raise TypeError("scipy distance metrics do not"
                             " support sparse matrices.")
-        X, Y = check_pairwise_arrays(X, Y)
+
+        dtype = bool if metric in PAIRWISE_BOOLEAN_FUNCTIONS else None
+
+        X, Y = check_pairwise_arrays(X, Y, dtype=dtype)
+
         if n_jobs == 1 and X is Y:
             return distance.squareform(distance.pdist(X, metric=metric,
                                                       **kwds))
@@ -1217,6 +1235,20 @@ def pairwise_distances(X, Y=None, metric="euclidean", n_jobs=1, **kwds):
     return _parallel_pairwise(X, Y, func, n_jobs, **kwds)
 
 
+# These distances recquire boolean arrays, when using scipy.spatial.distance
+PAIRWISE_BOOLEAN_FUNCTIONS = [
+    'dice',
+    'jaccard',
+    'kulsinski',
+    'matching',
+    'rogerstanimoto',
+    'russellrao',
+    'sokalmichener',
+    'sokalsneath',
+    'yule',
+]
+
+
 # Helper functions - distance
 PAIRWISE_KERNEL_FUNCTIONS = {
     # If updating this dictionary, update the doc in both distance_metrics()

sklearn/metrics/tests/test_pairwise.py

@@ -12,6 +12,7 @@
 from sklearn.utils.testing import assert_raises
 from sklearn.utils.testing import assert_raises_regexp
 from sklearn.utils.testing import assert_true
+from sklearn.utils.testing import ignore_warnings
 
 from sklearn.externals.six import iteritems
 
@@ -31,13 +32,15 @@
 from sklearn.metrics.pairwise import pairwise_kernels
 from sklearn.metrics.pairwise import PAIRWISE_KERNEL_FUNCTIONS
 from sklearn.metrics.pairwise import PAIRWISE_DISTANCE_FUNCTIONS
+from sklearn.metrics.pairwise import PAIRWISE_BOOLEAN_FUNCTIONS
 from sklearn.metrics.pairwise import PAIRED_DISTANCES
 from sklearn.metrics.pairwise import check_pairwise_arrays
 from sklearn.metrics.pairwise import check_paired_arrays
 from sklearn.metrics.pairwise import paired_distances
 from sklearn.metrics.pairwise import paired_euclidean_distances
 from sklearn.metrics.pairwise import paired_manhattan_distances
 from sklearn.preprocessing import normalize
+from sklearn.exceptions import DataConversionWarning
 
 
 def test_pairwise_distances():
@@ -115,6 +118,22 @@ def test_pairwise_distances():
     assert_raises(ValueError, pairwise_distances, X, Y, metric="blah")
 
 
+# ignore conversion to boolean in pairwise_distances
+@ignore_warnings(category=DataConversionWarning)
+def test_pairwise_boolean_distance():
+    # test that we convert to boolean arrays for boolean distances
+    rng = np.random.RandomState(0)
+    X = rng.randn(5, 4)
+    Y = X.copy()
+    Y[0, 0] = 1 - Y[0, 0]
+
+    for metric in PAIRWISE_BOOLEAN_FUNCTIONS:
+        for Z in [Y, None]:
+            res = pairwise_distances(X, Z, metric=metric)
+            res[np.isnan(res)] = 0
+            assert_true(np.sum(res != 0) == 0)
+
+
 def test_pairwise_precomputed():
     for func in [pairwise_distances, pairwise_kernels]:
         # Test correct shape
@@ -143,7 +162,7 @@ def test_pairwise_precomputed():
         assert_equal('f', S.dtype.kind)
 
         # Test converts list to array-like
-        S = func([[1]], metric='precomputed')
+        S = func([[1.]], metric='precomputed')
         assert_true(isinstance(S, np.ndarray))
 
 
@@ -188,7 +207,7 @@ def test_pairwise_callable_nonstrict_metric():
     # paired_distances should allow callable metric where metric(x, x) != 0
     # Knowing that the callable is a strict metric would allow the diagonal to
     # be left uncalculated and set to 0.
-    assert_equal(pairwise_distances([[1]], metric=lambda x, y: 5)[0, 0], 5)
+    assert_equal(pairwise_distances([[1.]], metric=lambda x, y: 5)[0, 0], 5)
 
 
 def callable_rbf_kernel(x, y, **kwds):

sklearn/neighbors/tests/test_neighbors.py

@@ -17,6 +17,7 @@
 from sklearn.utils.validation import check_random_state
 from sklearn.metrics.pairwise import pairwise_distances
 from sklearn import neighbors, datasets
+from sklearn.exceptions import DataConversionWarning
 
 rng = np.random.RandomState(0)
 # load and shuffle iris dataset
@@ -1200,3 +1201,18 @@ def test_dtype_convert():
 
     result = classifier.fit(X, y).predict(X)
     assert_array_equal(result, y)
+
+
+# ignore conversion to boolean in pairwise_distances
+@ignore_warnings(category=DataConversionWarning)
+def test_pairwise_boolean_distance():
+    # Non-regression test for #4523
+    # 'brute': uses scipy.spatial.distance through pairwise_distances
+    # 'ball_tree': uses sklearn.neighbors.dist_metrics
+    rng = np.random.RandomState(0)
+    X = rng.uniform(size=(6, 5))
+    NN = neighbors.NearestNeighbors
+
+    nn1 = NN(metric="jaccard", algorithm='brute').fit(X)
+    nn2 = NN(metric="jaccard", algorithm='ball_tree').fit(X)
+    assert_array_equal(nn1.kneighbors(X)[0], nn2.kneighbors(X)[0])