ENH Use dense arrays throughout

Introducing sparse matrices is an unnecessary complication

Author: luispedro

ch08/corrneighbours.py

@@ -49,15 +49,13 @@ def train_test(user, rest):
 
 
 def all_estimates(reviews):
-    reviews = reviews.toarray()
     estimates = np.zeros_like(reviews)
     for i in range(reviews.shape[0]):
         estimates[i] = estimate_user(reviews[i], np.delete(reviews, i, 0))
     return estimates
 
 def main():
     reviews = load()
-    reviews = reviews.toarray()
 
     err = []
     for i in range(reviews.shape[0]):
@@ -67,11 +65,16 @@ def main():
     revs = (reviews > 0).sum(1)
     err = np.array(err)
     rmse = np.sqrt(err / revs[:, None])
+
+    rmse_model, rmse_null = np.mean(rmse, 0)
+
     print("Average of RMSE / Null-model RMSE")
-    print(np.mean(rmse, 0))
+    print("{:.2}\t{:.2} (improvement: {:.1%}".format(rmse_model, rmse_null, (rmse_null-rmse_model)/rmse_null))
     print()
+
+    rmse_model, rmse_null = np.mean(rmse[revs > 60], 0)
     print("Average of RMSE / Null-model RMSE (users with more than 60 reviewed movies)")
-    print(np.mean(rmse[revs > 60], 0))
+    print("{:.2}\t{:.2} (improvement: {:.1%}".format(rmse_model, rmse_null, (rmse_null-rmse_model)/rmse_null))
 
 if __name__ == '__main__':
     main()

ch08/figure3.py

@@ -8,7 +8,6 @@
 from load_ml100k import load
 from matplotlib import pyplot as plt
 data = load()
-data = data.toarray()
 plt.gray()
 plt.imshow(data[:200, :200], interpolation='nearest')
 plt.xlabel('User ID')

ch08/load_ml100k.py

@@ -17,4 +17,4 @@ def load():
     ij -= 1  # original data is in 1-based system
     values = data[:, 2]
     reviews = sparse.csc_matrix((values, ij.T)).astype(float)
-    return reviews
+    return reviews.toarray()

ch08/similar_movie.py

@@ -65,7 +65,7 @@ def all_estimates(reviews, k=1):
 
 if __name__ == '__main__':
     from load_ml100k import load
-    reviews = load().torarray()
+    reviews = load()
     estimates = all_estimates(reviews)
     error = (estimates - reviews)
     error **= 2

ch08/stacked.py

@@ -21,9 +21,6 @@
     similar_movie.all_estimates(reviews),
 ])
 
-reviews = reviews.toarray()
-
-
 total_error = 0.0
 coefficients = []
 for u in range(reviews.shape[0]):

ch08/stacked5.py

@@ -11,11 +11,10 @@
 import similar_movie
 import usermodel
 
-sreviews = load()
-reviews = sreviews.toarray()
+reviews = load()
 # Collect several estimates
 es = np.array([
-    usermodel.all_estimates(sreviews),
+    usermodel.all_estimates(reviews),
     similar_movie.all_estimates(reviews, k=1),
     similar_movie.all_estimates(reviews, k=2),
     similar_movie.all_estimates(reviews, k=3),

ch08/usermodel.py

@@ -6,42 +6,43 @@
 # It is made available under the MIT License
 
 import numpy as np
-from sklearn.linear_model import LassoCV, RidgeCV, ElasticNetCV
+from sklearn.linear_model import ElasticNetCV
 from sklearn.cross_validation import KFold
-from load_ml100k import load
 
 
 def learn_for(reviews, i):
     reg = ElasticNetCV(fit_intercept=True, alphas=[
                        0.0125, 0.025, 0.05, .125, .25, .5, 1., 2., 4.])
+    nusers,nmovies = reviews.shape
     u = reviews[i]
-    us = range(reviews.shape[0])
-    del us[i]
-    ps, = np.where(u.toarray().ravel() > 0)
+    us = np.arange(reviews.shape[0])
+    us = np.delete(us, i)
+    ps, = np.where(u.ravel() > 0)
     x = reviews[us][:, ps].T
-    y = u.data
-    kf = KFold(len(y), n_folds=4)
+    kf = KFold(len(ps), n_folds=4)
     predictions = np.zeros(len(ps))
     for train, test in kf:
-        xc = x[train].copy().toarray()
+        xc = x[train].copy()
         x1 = np.array([xi[xi > 0].mean() for xi in xc])
         x1 = np.nan_to_num(x1)
 
         for i in range(xc.shape[0]):
             xc[i] -= (xc[i] > 0) * x1[i]
 
-        reg.fit(xc, y[train] - x1)
+        reg.fit(xc, u[train] - x1)
 
-        xc = x[test].copy().toarray()
+        xc = x[test].copy()
         x1 = np.array([xi[xi > 0].mean() for xi in xc])
         x1 = np.nan_to_num(x1)
 
         for i in range(xc.shape[0]):
             xc[i] -= (xc[i] > 0) * x1[i]
 
-        p = np.array(map(reg.predict, xc)).ravel()
+        p = reg.predict(xc).ravel()
         predictions[test] = p
-    return predictions
+    fill_preds = np.zeros(nmovies)
+    fill_preds[ps] = predictions
+    return fill_preds
 
 
 def all_estimates(reviews):
@@ -50,3 +51,4 @@ def all_estimates(reviews):
         s = learn_for(reviews, i)
         whole_data.append(s)
     return np.array(whole_data)
+