viz to chp3, plus github example scrapper + csv

Author: CamDavidsonPilon

Chapter3_MCMC/IntroMCMC.ipynb

@@ -0,0 +1,121 @@
+Python, Javascript, Ruby, Java, Shell, PHP, days_since_creation, has_wiki, author_following, author_followers, starred_count, forked_count
+0,1,0,0,0,0,523,1,0,0,2,1
+0,1,0,0,0,0,778,1,193,95,1,0
+0,0,0,1,0,0,531,1,1,4,3,0
+0,0,1,0,0,0,396,1,1,2,2,0
+0,0,0,0,1,0,846,1,3,5,1,0
+0,0,1,0,0,0,520,1,155,56,1,0
+0,1,0,0,0,0,384,1,0,0,2,0
+1,0,0,0,0,0,705,1,33,9,1,0
+0,0,0,0,0,1,726,1,0,7,11,1
+1,0,0,0,0,0,413,1,0,2,1,0
+1,0,0,0,0,0,141,1,3,0,1,0
+1,0,0,0,0,0,1410,1,65,82,1,0
+0,0,0,0,0,1,175,1,0,0,0,0
+0,0,1,0,0,0,365,1,49,85,1,0
+0,0,0,0,0,1,580,1,9,4,1,0
+0,0,0,1,0,0,471,1,1,0,1,0
+0,1,0,0,0,0,353,1,5,2,2,1
+0,0,1,0,0,0,973,1,0,0,1,0
+0,0,1,0,0,0,591,1,0,0,1,0
+0,0,1,0,0,0,85,1,0,0,0,0
+0,1,0,0,0,0,431,1,0,0,1,0
+1,0,0,0,0,0,188,1,1,1,0,0
+0,0,1,0,0,0,144,1,0,0,0,0
+0,1,0,0,0,0,102,1,0,0,0,0
+0,1,0,0,0,0,249,1,0,0,2,0
+0,1,0,0,0,0,395,1,2,0,1,0
+0,1,0,0,0,0,433,1,12,69,1,0
+0,0,0,1,0,0,615,1,0,0,1,0
+0,0,1,0,0,0,707,1,0,0,1,0
+0,0,0,1,0,0,256,1,0,1,1,0
+0,0,1,0,0,0,363,1,0,0,1,0
+0,0,1,0,0,0,1818,1,98,162,16,6
+0,1,0,0,0,0,86,1,13,7,0,0
+0,1,0,0,0,0,284,1,1,3,2,1
+0,0,0,1,0,0,867,1,9,5,2,0
+0,0,1,0,0,0,1203,1,1,0,1,0
+0,1,0,0,0,0,436,1,34,4,0,0
+0,1,0,0,0,0,720,1,6,73,6,0
+0,0,1,0,0,0,948,1,0,84,18,4
+1,0,0,0,0,0,383,1,0,0,2,1
+0,0,0,1,0,0,62,1,0,0,0,0
+0,0,0,1,0,0,927,1,0,0,40,17
+0,1,0,0,0,0,1303,1,0,0,3,0
+0,1,0,0,0,0,113,1,0,0,0,0
+0,0,1,0,0,0,810,1,4,5,1,0
+0,0,0,0,0,1,43,1,7,11,2,0
+0,0,0,0,0,1,369,1,0,0,1,0
+0,1,0,0,0,0,449,1,106,4398,399,60
+0,1,0,0,0,0,200,1,0,0,0,0
+0,1,0,0,0,0,122,1,4,2,0,0
+0,0,1,0,0,0,738,1,0,0,1,0
+0,1,0,0,0,0,705,1,0,0,2,0
+0,0,1,0,0,0,291,1,0,0,1,0
+0,1,0,0,0,0,621,1,3,23,1,0
+0,0,1,0,0,0,368,1,0,0,1,0
+0,0,1,0,0,0,851,1,0,0,1,0
+0,0,0,1,0,0,58,1,0,0,0,0
+0,0,0,1,0,0,706,1,0,8,2,1
+0,0,0,0,0,1,817,1,0,1,1,1
+0,0,0,0,1,0,1475,1,0,4,1,0
+0,0,1,0,0,0,584,1,0,0,1,0
+1,0,0,0,0,0,841,1,0,7,1,0
+0,0,0,0,0,1,382,1,0,74,1,0
+0,0,1,0,0,0,213,1,1,0,1,0
+0,0,1,0,0,0,93,1,0,0,0,0
+0,1,0,0,0,0,356,1,1,10,2,0
+0,0,1,0,0,0,123,1,0,1,0,0
+0,0,1,0,0,0,113,1,35,63,0,0
+0,0,0,0,0,1,302,1,13,2,3,1
+0,1,0,0,0,0,550,0,19,18,2,0
+0,0,1,0,0,0,721,1,2,0,1,0
+0,0,1,0,0,0,160,1,5,6,11,6
+1,0,0,0,0,0,399,1,7,6,1,0
+0,1,0,0,0,0,206,1,15,116,2,0
+0,0,0,1,0,0,319,1,0,0,2,1
+0,1,0,0,0,0,111,1,2,3,0,0
+0,0,0,0,0,1,632,1,10,3,2,1
+0,0,0,0,0,1,1044,1,16,23,3,1
+0,0,0,0,0,1,194,1,0,0,0,0
+0,1,0,0,0,0,908,1,56,503,43,8
+0,0,1,0,0,0,89,1,0,0,0,0
+0,1,0,0,0,0,1311,1,107,119,4,1
+0,1,0,0,0,0,78,1,0,0,0,0
+0,0,1,0,0,0,564,1,0,2,1,0
+0,0,1,0,0,0,1049,1,0,0,1,0
+0,0,0,1,0,0,322,1,0,0,2,1
+0,0,1,0,0,0,1443,1,11,7,3,8
+0,0,1,0,0,0,310,1,1,0,1,0
+0,0,0,0,0,1,515,1,0,2,1,0
+0,0,1,0,0,0,563,1,0,0,1,0
+1,0,0,0,0,0,344,1,0,0,1,0
+0,1,0,0,0,0,132,1,2,2,0,0
+0,0,1,0,0,0,837,1,0,0,1,0
+0,1,0,0,0,0,459,1,1,4,1,0
+0,0,0,0,0,1,768,1,8,2,3,0
+0,0,0,0,0,1,410,1,0,4,2,1
+1,0,0,0,0,0,380,1,0,0,9,0
+0,0,0,1,0,0,859,1,14,22,1,0
+1,0,0,0,0,0,628,1,0,1,1,0
+0,0,1,0,0,0,206,0,55,50,21,2
+0,0,0,1,0,0,117,1,1,2,0,0
+0,0,0,0,1,0,237,1,0,0,1,0
+0,0,1,0,0,0,462,1,0,1,1,0
+1,0,0,0,0,0,628,1,2,3,1,0
+0,0,0,1,0,0,504,1,1,6,1,0
+0,0,1,0,0,0,283,1,0,0,1,0
+0,1,0,0,0,0,654,0,4,7,1,0
+0,1,0,0,0,0,97,1,0,0,0,0
+1,0,0,0,0,0,396,1,3,11,1,0
+0,0,0,0,1,0,179,1,0,0,0,0
+0,1,0,0,0,0,1743,1,3,0,1,0
+0,0,1,0,0,0,949,1,1,4,75,24
+0,0,0,1,0,0,307,1,0,0,1,1
+0,0,1,0,0,0,264,1,0,0,1,0
+0,0,1,0,0,0,193,1,4,5,0,0
+0,0,0,0,1,0,1148,1,4,37,2,1
+0,0,1,0,0,0,150,1,0,0,0,0
+0,0,0,0,1,0,250,1,19,9,1,0
+0,0,1,0,0,0,1726,1,7,287,1,0
+0,1,0,0,0,0,340,1,0,0,1,0

Chapter3_MCMC/data/github_data.csv

@@ -0,0 +1,88 @@
+#github data scrapper
+
+"""
+variables of interest:
+    indp. variables
+    - language, given as a binary variable. Need 4 positions for 5 langagues
+    - #number of days created ago, 1 position
+    - has wiki? Boolean, 1 position
+    - followers, 1 position
+    - following, 1 position
+    - constant
+    
+    dep. variables
+    -stars/watchers
+    -forks
+
+"""
+from requests import get
+from json import loads
+import datetime
+import numpy as np
+
+
+MAX = 8000000
+today =  datetime.datetime.today()
+randint = np.random.randint
+N = 120 #sample size. 
+auth = ("username", "password" )
+
+language_mappings = {"Python": 0, "JavaScript": 1, "Ruby": 2, "Java":3, "Shell":4, "PHP":5}
+
+#define data matrix: 
+X = np.zeros( (N , 12), dtype = int )
+
+for i in xrange(N):
+    is_fork = True
+    is_valid_language = False
+    
+    while is_fork == True or is_valid_language == False:
+        is_fork = True
+        is_valid_language = False
+        
+        params = {"since":randint(0, MAX ) }
+        r = get("https://api.github.com/repositories", params = params, auth=auth )
+        results = loads( r.text )[0]
+        #im only interested in the first one, and if it is not a fork.
+        is_fork = results["fork"]
+        
+        r = get( results["url"], auth = auth)
+        
+        #check the language
+        repo_results = loads( r.text )
+        try: 
+            language_mappings[ repo_results["language" ] ]
+            is_valid_language = True
+        except:
+            pass
+        
+    
+
+    #languages 
+    X[ i, language_mappings[ repo_results["language" ] ] ] = 1
+    
+    #delta time
+    X[ i, 6] = ( today - datetime.datetime.strptime( repo_results["created_at"][:10], "%Y-%m-%d" ) ).days
+    
+    #haswiki
+    X[i, 7] = repo_results["has_wiki"]
+    
+    #get user information
+    r = get( results["owner"]["url"] , auth = auth)
+    user_results = loads( r.text )
+    X[i, 8] = user_results["following"]
+    X[i, 9] = user_results["followers"]
+    
+    #get dep. data
+    X[i, 10] = repo_results["watchers_count"]
+    X[i, 11] = repo_results["forks_count"]
+    print 
+    print " -------------- "
+    print i, ": ", results["full_name"], repo_results["language" ], repo_results["watchers_count"], repo_results["forks_count"]
+    print " -------------- "
+    print 
+    
+np.savetxt("data/github_data.csv", X, delimiter=",", fmt="%d" )
+    
+
+