Merge pull request scikit-learn#4840 from jnothman/dynamicgrid

[MRG] FIX avoid memory cost when sampling from large parameter grids

Author: jnothman

doc/whats_new.rst

@@ -97,6 +97,9 @@ Bug fixes
     - Fixed bug in :class:`decomposition.DictLearning` when ``n_jobs < 0``. By
       `Andreas Müller`_.
 
+    - Fixed bug where :class:`grid_search.RandomizedSearchCV` could consume a
+      lot of memory for large discrete grids. By `Joel Nothman`_.
+
 API changes summary
 -------------------
 

sklearn/grid_search.py

@@ -70,6 +70,8 @@ class ParameterGrid(object):
     ...                               {'kernel': 'rbf', 'gamma': 1},
     ...                               {'kernel': 'rbf', 'gamma': 10}]
     True
+    >>> ParameterGrid(grid)[1] == {'kernel': 'rbf', 'gamma': 1}
+    True
 
     See also
     --------
@@ -111,6 +113,47 @@ def __len__(self):
         return sum(product(len(v) for v in p.values()) if p else 1
                    for p in self.param_grid)
 
+    def __getitem__(self, ind):
+        """Get the parameters that would be ``ind``th in iteration
+
+        Parameters
+        ----------
+        ind : int
+            The iteration index
+
+        Returns
+        -------
+        params : dict of string to any
+            Equal to list(self)[ind]
+        """
+        # This is used to make discrete sampling without replacement memory
+        # efficient.
+        for sub_grid in self.param_grid:
+            # XXX: could memoize information used here
+            if not sub_grid:
+                if ind == 0:
+                    return {}
+                else:
+                    ind -= 1
+                    continue
+
+            # Reverse so most frequent cycling parameter comes first
+            keys, values_lists = zip(*sorted(sub_grid.items())[::-1])
+            sizes = [len(v_list) for v_list in values_lists]
+            total = np.product(sizes)
+
+            if ind >= total:
+                # Try the next grid
+                ind -= total
+            else:
+                out = {}
+                for key, v_list, n in zip(keys, values_lists, sizes):
+                    ind, offset = divmod(ind, n)
+                    out[key] = v_list[offset]
+                return out
+
+        raise IndexError('ParameterGrid index out of range')
+
 
 class ParameterSampler(object):
     """Generator on parameters sampled from given distributions.
@@ -181,8 +224,8 @@ def __iter__(self):
         rnd = check_random_state(self.random_state)
 
         if all_lists:
-            # get complete grid and yield from it
-            param_grid = list(ParameterGrid(self.param_distributions))
+            # look up sampled parameter settings in parameter grid
+            param_grid = ParameterGrid(self.param_distributions)
             grid_size = len(param_grid)
 
             if grid_size < self.n_iter:

sklearn/tests/test_grid_search.py

@@ -92,13 +92,18 @@ def score(self):
 y = np.array([1, 1, 2, 2])
 
 
+def assert_grid_iter_equals_getitem(grid):
+    assert_equal(list(grid), [grid[i] for i in range(len(grid))])
+
+
 def test_parameter_grid():
     # Test basic properties of ParameterGrid.
     params1 = {"foo": [1, 2, 3]}
     grid1 = ParameterGrid(params1)
     assert_true(isinstance(grid1, Iterable))
     assert_true(isinstance(grid1, Sized))
     assert_equal(len(grid1), 3)
+    assert_grid_iter_equals_getitem(grid1)
 
     params2 = {"foo": [4, 2],
                "bar": ["ham", "spam", "eggs"]}
@@ -113,14 +118,19 @@ def test_parameter_grid():
                      set(("bar", x, "foo", y)
                          for x, y in product(params2["bar"], params2["foo"])))
 
+    assert_grid_iter_equals_getitem(grid2)
+
     # Special case: empty grid (useful to get default estimator settings)
     empty = ParameterGrid({})
     assert_equal(len(empty), 1)
     assert_equal(list(empty), [{}])
+    assert_grid_iter_equals_getitem(empty)
+    assert_raises(IndexError, lambda: empty[1])
 
-    has_empty = ParameterGrid([{'C': [1, 10]}, {}])
-    assert_equal(len(has_empty), 3)
-    assert_equal(list(has_empty), [{'C': 1}, {'C': 10}, {}])
+    has_empty = ParameterGrid([{'C': [1, 10]}, {}, {'C': [.5]}])
+    assert_equal(len(has_empty), 4)
+    assert_equal(list(has_empty), [{'C': 1}, {'C': 10}, {}, {'C': .5}])
+    assert_grid_iter_equals_getitem(has_empty)
 
 
 def test_grid_search():