ENH add VarianceThreshold feature selection method

Author: larsmans

doc/modules/feature_selection.rst

@@ -12,6 +12,39 @@ for feature selection/dimensionality reduction on sample sets, either to
 improve estimators' accuracy scores or to boost their performance on very
 high-dimensional datasets.
 
+
+Removing features with low variance
+===================================
+
+:class:`VarianceThreshold` is a simple baseline approach to feature selection.
+It removes all features whose variance doesn't meet some threshold.
+By default, it removes all zero-variance features,
+i.e. features that have the same value in all samples.
+
+As an example, suppose that we have a dataset with boolean features,
+and we want to remove all features that are either one or zero (on or off)
+in more than 80% of the samples.
+Boolean features are Bernoulli random variables,
+and the variance of such variables is given by
+
+.. math:: \mathrm{Var}[X] = p(1 - p)
+
+so we can select using the threshold ``.8 * (1 - .8)``::
+
+  >>> from sklearn.feature_selection import VarianceThreshold
+  >>> X = [[0, 0, 1], [0, 1, 0], [1, 0, 0], [0, 1, 1], [0, 1, 0], [0, 1, 1]]
+  >>> sel = VarianceThreshold(threshold=(.8 * (1 - .8)))
+  >>> sel.fit_transform(X)
+  array([[0, 1],
+         [1, 0],
+         [0, 0],
+         [1, 1],
+         [1, 0],
+         [1, 1]])
+
+As expected, ``VarianceThreshold`` has removed the first column,
+which has a probability :math:`p = 5/6 > .8` of containing a one.
+
 Univariate feature selection
 ============================
 

doc/whats_new.rst

@@ -5,9 +5,15 @@
 0.15
 =====
 
+Changelog
+---------
+
    - Add predict method to :class:`cluster.AffinityPropagation` and
      :class:`cluster.MeanShift`, by `Mathieu Blondel`_.
 
+   - New unsupervised feature selection algorithm
+     :class:`feature_selection.VarianceThreshold`, by `Lars Buitinck`_.
+
 .. _changes_0_14:
 
 0.14

sklearn/feature_selection/__init__.py

@@ -15,6 +15,8 @@
 from .univariate_selection import SelectFwe
 from .univariate_selection import GenericUnivariateSelect
 
+from .variance_threshold import VarianceThreshold
+
 from .rfe import RFE
 from .rfe import RFECV
 

sklearn/feature_selection/tests/test_variance_threshold.py

@@ -0,0 +1,29 @@
+from sklearn.utils.testing import (assert_array_equal, assert_equal,
+                                   assert_raises)
+
+import numpy as np
+from scipy.sparse import bsr_matrix, csc_matrix, csr_matrix
+
+from sklearn.feature_selection import VarianceThreshold
+
+data = [[0, 1, 2, 3, 4],
+        [0, 2, 2, 3, 5],
+        [1, 1, 2, 4, 0]]
+
+
+def test_zero_variance():
+    """Test VarianceThreshold with default setting, zero variance."""
+
+    for X in [data, csr_matrix(data), csc_matrix(data), bsr_matrix(data)]:
+        sel = VarianceThreshold().fit(X)
+        assert_array_equal([0, 1, 3, 4], sel.get_support(indices=True))
+
+    assert_raises(ValueError, VarianceThreshold().fit, [0, 1, 2, 3])
+    assert_raises(ValueError, VarianceThreshold().fit, [[0, 1], [0, 1]])
+
+
+def test_variance_threshold():
+    """Test VarianceThreshold with custom variance."""
+    for X in [data, csr_matrix(data)]:
+        X = VarianceThreshold(threshold=.4).fit_transform(X)
+        assert_equal((len(data), 1), X.shape)

sklearn/feature_selection/variance_threshold.py

@@ -0,0 +1,77 @@
+# Author: Lars Buitinck <[email protected]>
+# License: 3-clause BSD
+
+import numpy as np
+from ..base import BaseEstimator
+from .base import SelectorMixin
+from ..utils import atleast2d_or_csr
+from ..utils.sparsefuncs import csr_mean_variance_axis0
+
+
+class VarianceThreshold(BaseEstimator, SelectorMixin):
+    """Feature selector that removes all low-variance features.
+
+    This feature selection algorithm looks only at the features (X), not the
+    desired outputs (y), and can thus be used for unsupervised learning.
+
+    Parameters
+    ----------
+    threshold : float, optional
+        Features with a training-set variance lower than this threshold will
+        be removed. The default is to keep all features with non-zero variance,
+        i.e. remove the features that have the same value in all samples.
+
+    Attributes
+    ----------
+    `variances_` : array, shape (n_features,)
+        Variances of individual features.
+
+    Examples
+    --------
+    The following dataset has integer features, two of which are the same
+    in every sample. These are removed with the default setting for threshold::
+
+        >>> X = [[0, 2, 0, 3], [0, 1, 4, 3], [0, 1, 1, 3]]
+        >>> selector = VarianceThreshold()
+        >>> selector.fit_transform(X)
+        array([[2, 0],
+               [1, 4],
+               [1, 1]])
+    """
+
+    def __init__(self, threshold=0.):
+        self.threshold = threshold
+
+    def fit(self, X, y=None):
+        """Learn empirical variances from X.
+
+        Parameters
+        ----------
+        X : {array-like, sparse matrix}, shape (n_samples, n_features)
+            Sample vectors from which to compute variances.
+
+        y : any
+            Ignored. This parameter exists only for compatibility with
+            sklearn.pipeline.Pipeline.
+
+        Returns
+        -------
+        self
+        """
+        X = atleast2d_or_csr(X, dtype=np.float64)
+
+        if hasattr(X, "toarray"):   # sparse matrix
+            _, self.variances_ = csr_mean_variance_axis0(X)
+        else:
+            self.variances_ = np.var(X, axis=0)
+
+        if np.all(self.variances_ <= self.threshold):
+            msg = "No feature in X meets the variance threshold {0:.5f}"
+            if X.shape[0] == 1:
+                msg += " (X contains only one sample)"
+            raise ValueError(msg.format(self.threshold))
+
+        return self
+
+    def _get_support_mask(self):
+        return self.variances_ > self.threshold