ENH reduce memory usage of DictVectorizer.transform in sparse case

Don't materialize iterable's entire contents.

Author: larsmans

sklearn/feature_extraction/dict_vectorizer.py

@@ -175,8 +175,6 @@ def transform(self, X, y=None):
         Xa : {array, sparse matrix}
             Feature vectors; always 2-d.
         """
-        X = _tosequence(X)
-
         dtype = self.dtype
         vocab = self.vocabulary_
 
@@ -198,10 +196,12 @@ def transform(self, X, y=None):
                     except KeyError:
                         pass
 
+            shape = (i + 1, len(vocab))
             return sp.coo_matrix((values, (i_ind, j_ind)),
-                                 shape=(len(X), len(vocab)), dtype=dtype)
+                                 shape=shape, dtype=dtype)
 
         else:
+            X = _tosequence(X)
             Xa = np.zeros((len(X), len(vocab)), dtype=dtype)
 
             for i, x in enumerate(X):

sklearn/feature_extraction/tests/test_dict_vectorizer.py

@@ -26,6 +26,12 @@ def test_dictvectorizer():
             assert_equal(X.sum(), 14)
             assert_equal(v.inverse_transform(X), D)
 
+            if sparse:
+                # COO matrices can't be compared for equality
+                assert_array_equal(X.A, v.transform(D).A)
+            else:
+                assert_array_equal(X, v.transform(D))
+
 
 def test_feature_selection():
     # make two feature dicts with two useful features and a bunch of useless