diff --git a/.gitignore b/.gitignore index 449fe6d..87031a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ .pyc .DS_Store -_tmp \ No newline at end of file +_tmp +env +venv +__pycache__ +.env diff --git a/08_basic_email_web_crawler.py b/08_basic_email_web_crawler.py deleted file mode 100644 index faca75f..0000000 --- a/08_basic_email_web_crawler.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -import re -import urlparse - -# regex -email_re = re.compile(r'([\w\.,]+@[\w\.,]+\.\w+)') -link_re = re.compile(r'href="https://github.com/(.*?)"') - - -def crawl(url): - - result = set() - - req = requests.get(url) - - # Check if successful - if(req.status_code != 200): - return [] - - # Find links - links = link_re.findall(req.text) - - print "\nFound {} links".format(len(links)) - - # Search links for emails - for link in links: - - # Get an absolute URL for a link - link = urlparse.urljoin(url, link) - - # Find all emails on current page - result.update(email_re.findall(req.text)) - - return result - -if __name__ == '__main__': - emails = crawl('http://www.realpython.com/') - - print "\nScrapped e-mail addresses:" - for email in emails: - print email - print "\n" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3a6fa11 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Real Python + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..9d2dfea --- /dev/null +++ b/TODO.md @@ -0,0 +1,5 @@ +1. Write unit and integration tests for *all* scripts +1. Add Travis +1. Add support for Python 2.7, 3.5, and 3.6 +1. Organize docs and folder structure better +1. Add all scripts to single CLI for easy running, testing, and searching diff --git a/readme.md b/readme.md old mode 100644 new mode 100755 index ee46cd8..78f4a72 --- a/readme.md +++ b/readme.md @@ -4,7 +4,7 @@ 1. **02_find_all_links.py**: get all links from a webpage 1. **03_simple_twitter_manager.py**: accessing the Twitter API, example functions 1. **04_rename_with_slice.py**: rename group of files, within a single directory, using slice -1. **05_load_json_without_dupes.py**: load json, convert to dict, raise error if there is a duplicate key +1. **05_load_json_without_dupes.py**: load JSON, convert to dict, raise error if there is a duplicate key 1. **06_execution_time.py**: class used for timing execution of code 1. **07_benchmark_permissions_loading_django.py**: benchmark loading of permissions in Django 1. **08_basic_email_web_crawler.py**: web crawler for grabbing emails from a website @@ -19,4 +19,18 @@ 1. **17_rewrite_git_history.md**: Backdating/Rewriting Git history (use at your own risk) 1. **18_zipper.py**: Zip contents of a directory, adding a timestamp to the filename 1. **19_tsv-to-csv.py**: Convert TSV to CSV -1. **20_restore_file_from_git.py**: Restore file from Git History \ No newline at end of file +1. **20_restore_file_from_git.py**: Restore file from Git History +1. **21_twitter_bot.py**: Twitter Bot +1. **22_git_tag.py**: Create Git Tag based on a commit +1. **23_flask_session_test.py**: Just a simple app to see if the sessions are working +1. **24_sql2csv.py**: SQL to CSV. +1. **25_ip2geolocation.py**: Given a CSV file with an ip address (see sample - *25_sample_csv.csv*), return the geolocation based on the ip. +1. **26_stock_scraper.py**: Scrape the S&P 500 Companies list from Wikipedia, then output the data. +1. **27_send_sms.py**: Send SMS message via [TextBelt](http://textbelt.com/) +1. **28_income_tax_calculator.py**: Income tax calculator via [Taxee](http://taxee.io/) +1. **29_json_to_yaml.py**: Convert JSON to YAML +1. **30_fullcontact.py**: Call the [FullcContact](https://www.fullcontact.com/developer/) API +1. **31_youtube_sentiment.py**: Calculate sentiment score from the comments of a Youtube video +1. **32_stock_scraper.py**: Get stock prices +1. **33_country_code.py**: Convert country code to country name +1. **34_git_all_repos.py**: Clone all repositories from a public user or organization on Github. Usage: `python git_all_repos.py users USER_NAME` or `python git_all_repos.py orgs ORG_NAME` diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..beee86c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +beautifulsoup4==4.4.1 +PyYAML==3.11 +requests==2.12.4 +wheel==0.24.0 +lxml==3.8.0 diff --git a/01_remove_all_pyc.md b/scripts/01_remove_all_pyc.md old mode 100644 new mode 100755 similarity index 66% rename from 01_remove_all_pyc.md rename to scripts/01_remove_all_pyc.md index 2891215..f666e1a --- a/01_remove_all_pyc.md +++ b/scripts/01_remove_all_pyc.md @@ -6,4 +6,4 @@ To recursively remove all those pesky *.pyc* files from a git repo, run this com $ find . -name "*.pyc" -exec git rm -f {} \; ``` -Then make sure to add a *.gitignore* and the root of the repo and add the line: `*.pyc` \ No newline at end of file +Then make sure to add a *.gitignore* in the root of the repo and add the line: `*.pyc` \ No newline at end of file diff --git a/02_find_all_links.py b/scripts/02_find_all_links.py old mode 100644 new mode 100755 similarity index 57% rename from 02_find_all_links.py rename to scripts/02_find_all_links.py index 76a7c99..37dff11 --- a/02_find_all_links.py +++ b/scripts/02_find_all_links.py @@ -1,18 +1,18 @@ -import urllib2 +import requests import re # get url -url =raw_input('Enter a URL (include `http://`): ') +url = input('Enter a URL (include `http://`): ') # connect to the url -website = urllib2.urlopen(url) +website = requests.get(url) # read html -html = website.read() +html = website.text # use re.findall to grab all the links links = re.findall('"((http|ftp)s?://.*?)"', html) # output links for link in links: - print link[0] \ No newline at end of file + print(link[0]) diff --git a/03_simple_twitter_manager.py b/scripts/03_simple_twitter_manager.py old mode 100644 new mode 100755 similarity index 64% rename from 03_simple_twitter_manager.py rename to scripts/03_simple_twitter_manager.py index b7e4a51..e39a20b --- a/03_simple_twitter_manager.py +++ b/scripts/03_simple_twitter_manager.py @@ -1,25 +1,28 @@ import twitter - - + + TWITTER_CONSUMER_KEY = 'XXX' TWITTER_CONSUMER_SECRET = 'XXX' TWITTER_ACCESS_TOKEN_KEY = 'XXX' TWITTER_ACCESS_TOKEN_SECRET = 'XXX' - + twitter_api = twitter.Api( consumer_key=TWITTER_CONSUMER_KEY, consumer_secret=TWITTER_CONSUMER_SECRET, access_token_key=TWITTER_ACCESS_TOKEN_KEY, access_token_secret=TWITTER_ACCESS_TOKEN_SECRET ) - + if __name__ == '__main__': follower_ids = twitter_api.GetFollowerIDs() following_ids = twitter_api.GetFriendIDs() - zombie_follows = [following_id for following_id in following_ids if following_id not in follower_ids] - - confirm = raw_input("Are you sure you want to unfollow %s tweeps [y|n]? " % (len(zombie_follows))) + zombie_follows = [following_id for following_id in + following_ids if following_id not in follower_ids] + + confirm = raw_input( + "Are you sure you want to unfollow {0} tweeps [y|n]? ".format( + (len(zombie_follows)))) if confirm.lower() == 'y': for id in zombie_follows: user = twitter_api.DestroyFriendship(user_id=id) - print "Unfollowed %s" % (user.screen_name) \ No newline at end of file + print("Unfollowed {0}".format(user.screen_name)) diff --git a/04_rename_with_slice.py b/scripts/04_rename_with_slice.py old mode 100644 new mode 100755 similarity index 75% rename from 04_rename_with_slice.py rename to scripts/04_rename_with_slice.py index 2ff84f7..dd849ef --- a/04_rename_with_slice.py +++ b/scripts/04_rename_with_slice.py @@ -8,7 +8,7 @@ new_file_name = file_name[:-6] + extension try: os.rename(file, new_file_name) - except OSError, e: - print e + except OSError as e: + print(e) else: - print "Renamed {} to {}".format(file, new_file_name) + print("Renamed {} to {}".format(file, new_file_name)) diff --git a/05_load_json_without_dupes.py b/scripts/05_load_json_without_dupes.py old mode 100644 new mode 100755 similarity index 58% rename from 05_load_json_without_dupes.py rename to scripts/05_load_json_without_dupes.py index 5b767a2..2cbe318 --- a/05_load_json_without_dupes.py +++ b/scripts/05_load_json_without_dupes.py @@ -1,11 +1,9 @@ -import json - def dict_raise_on_duplicates(ordered_pairs): """reject duplicate keys""" my_dict = dict() for key, values in ordered_pairs: if key in my_dict: - raise ValueError("Duplicate key: {}".format(key,)) + raise ValueError("Duplicate key: {}".format(key,)) else: - my_dict[key] = values - return my_dict \ No newline at end of file + my_dict[key] = values + return my_dict diff --git a/06_execution_time.py b/scripts/06_execution_time.py old mode 100644 new mode 100755 similarity index 75% rename from 06_execution_time.py rename to scripts/06_execution_time.py index 95d5ca4..9614bbd --- a/06_execution_time.py +++ b/scripts/06_execution_time.py @@ -13,6 +13,7 @@ import time +import random class ExecutionTime: @@ -25,9 +26,9 @@ def duration(self): # ---- run code ---- # -import random timer = ExecutionTime() sample_list = list() -my_list = [random.randint(1, 888898) for num in xrange(1, 1000000) if num % 2 == 0] -print 'Finished in {} seconds.'.format(timer.duration()) \ No newline at end of file +my_list = [random.randint(1, 888898) for num in + range(1, 1000000) if num % 2 == 0] +print('Finished in {} seconds.'.format(timer.duration())) diff --git a/07_benchmark_permissions_loading_django.py b/scripts/07_benchmark_permissions_loading_django.py old mode 100644 new mode 100755 similarity index 90% rename from 07_benchmark_permissions_loading_django.py rename to scripts/07_benchmark_permissions_loading_django.py index 0e2e06b..e1e6900 --- a/07_benchmark_permissions_loading_django.py +++ b/scripts/07_benchmark_permissions_loading_django.py @@ -14,8 +14,8 @@ def timed(*args, **kw): te = time.time() all_times.append(te - ts) - print all_times - print numpy.mean(all_times) + print(all_times) + print(numpy.mean(all_times)) return result return timed @@ -39,4 +39,4 @@ def load_new_perms(): while n < 10: create_new_db() load_new_perms() - n += 1 \ No newline at end of file + n += 1 diff --git a/scripts/08_basic_email_web_crawler.py b/scripts/08_basic_email_web_crawler.py new file mode 100755 index 0000000..b56c747 --- /dev/null +++ b/scripts/08_basic_email_web_crawler.py @@ -0,0 +1,21 @@ +import requests +import re + +# get url +url = input('Enter a URL (include `http://`): ') + +# connect to the url +website = requests.get(url) + +# read html +html = website.text + +# use re.findall to grab all the links +links = re.findall('"((http|ftp)s?://.*?)"', html) +emails = re.findall('([\w\.,]+@[\w\.,]+\.\w+)', html) + + +# print the number of links in the list +print("\nFound {} links".format(len(links))) +for email in emails: + print(email) diff --git a/09_basic_link_web_crawler.py b/scripts/09_basic_link_web_crawler.py old mode 100644 new mode 100755 similarity index 67% rename from 09_basic_link_web_crawler.py rename to scripts/09_basic_link_web_crawler.py index 4531ac3..87e2fab --- a/09_basic_link_web_crawler.py +++ b/scripts/09_basic_link_web_crawler.py @@ -1,6 +1,9 @@ import requests import re -import urlparse +try: + from urllib.parse import urljoin +except ImportError: + from urlparse import urljoin # regex link_re = re.compile(r'href="https://github.com/(.*?)"') @@ -17,17 +20,15 @@ def crawl(url): # Find links links = link_re.findall(req.text) - print "\nFound {} links".format(len(links)) + print("\nFound {} links".format(len(links))) # Search links for emails for link in links: # Get an absolute URL for a link - link = urlparse.urljoin(url, link) + link = urljoin(url, link) - print link - + print(link) if __name__ == '__main__': crawl('http://www.realpython.com/') - diff --git a/10_find_files_recursively.py b/scripts/10_find_files_recursively.py old mode 100644 new mode 100755 similarity index 69% rename from 10_find_files_recursively.py rename to scripts/10_find_files_recursively.py index 7251b10..0c8e1eb --- a/10_find_files_recursively.py +++ b/scripts/10_find_files_recursively.py @@ -2,8 +2,8 @@ import os # constants -PATH = '/../../../..' -PATTERN = '*.py' +PATH = './' +PATTERN = '*.md' def get_file_names(filepath, pattern): @@ -14,18 +14,18 @@ def get_file_names(filepath, pattern): # matches.append(os.path.join(root, filename)) # full path matches.append(os.path.join(filename)) # just file name if matches: - print "Found {} files:".format(len(matches)) + print("Found {} files:".format(len(matches))) output_files(matches) else: - print "No files found." + print("No files found.") else: - print "Sorry that path does not exist. Try again." + print("Sorry that path does not exist. Try again.") def output_files(list_of_files): for filename in list_of_files: - print filename + print(filename) if __name__ == '__main__': - all_files = get_file_names(PATH, PATTERN) \ No newline at end of file + get_file_names(PATH, PATTERN) diff --git a/11_optimize_images_with_wand.py b/scripts/11_optimize_images_with_wand.py old mode 100644 new mode 100755 similarity index 74% rename from 11_optimize_images_with_wand.py rename to scripts/11_optimize_images_with_wand.py index c3449fd..a95b8b0 --- a/11_optimize_images_with_wand.py +++ b/scripts/11_optimize_images_with_wand.py @@ -1,9 +1,9 @@ import fnmatch import os -# sudo pip install Wand +# pip install Wand from wand.image import Image -# sudo pip install http://pypi.python.org/packages/source/h/hurry.filesize/hurry.filesize-0.9.tar.gz +# pip install http://pypi.python.org/packages/source/h/hurry.filesize/hurry.filesize-0.9.tar.gz from hurry.filesize import size @@ -19,12 +19,13 @@ def get_image_file_names(filepath, pattern): for filename in fnmatch.filter(filenames, pattern): matches.append(os.path.join(root, filename)) # full path if matches: - print "Found {} files, with a total file size of {}.".format(len(matches), get_total_size(matches)) + print("Found {} files, with a total file size of {}.".format( + len(matches), get_total_size(matches))) return matches else: - print "No files found." + print("No files found.") else: - print "Sorry that path does not exist. Try again." + print("Sorry that path does not exist. Try again.") def get_total_size(list_of_image_names): @@ -35,7 +36,7 @@ def get_total_size(list_of_image_names): def resize_images(list_of_image_names): - print "Optimizing ... " + print("Optimizing ... ") for index, image_name in enumerate(list_of_image_names): with open(image_name) as f: image_binary = f.read() @@ -43,7 +44,7 @@ def resize_images(list_of_image_names): if img.height >= 600: img.transform(resize='x600') img.save(filename=image_name) - print "Optimization complete." + print("Optimization complete.") if __name__ == '__main__': diff --git a/12_csv_split.py b/scripts/12_csv_split.py old mode 100644 new mode 100755 similarity index 95% rename from 12_csv_split.py rename to scripts/12_csv_split.py index 65c698c..43ed1ee --- a/12_csv_split.py +++ b/scripts/12_csv_split.py @@ -117,10 +117,10 @@ def parse_file(arguments): writer = writer.writerows(chunk) # Output info - print "" - print "Chunk # {}:".format(current_chunk) - print "Filepath: {}".format(current_output) - print "# of rows: {}".format(len(chunk)) + print("") + print("Chunk # {}:".format(current_chunk)) + print("Filepath: {}".format(current_output)) + print("# of rows: {}".format(len(chunk))) # Create new chunk current_chunk += 1 diff --git a/12_sample_csv.csv b/scripts/12_sample_csv.csv old mode 100644 new mode 100755 similarity index 100% rename from 12_sample_csv.csv rename to scripts/12_sample_csv.csv diff --git a/13_random_name_generator.py b/scripts/13_random_name_generator.py old mode 100644 new mode 100755 similarity index 59% rename from 13_random_name_generator.py rename to scripts/13_random_name_generator.py index 0719eec..0acb9bf --- a/13_random_name_generator.py +++ b/scripts/13_random_name_generator.py @@ -1,4 +1,4 @@ -from random import randint +from random import choice def random_name_generator(first, second, x): @@ -10,17 +10,12 @@ def random_name_generator(first, second, x): - number of random names """ names = [] - for i in xrange(0, int(x)): - random_first = randint(0, len(first)-1) - random_last = randint(0, len(second)-1) - names.append("{0} {1}".format( - first[random_first], - second[random_last]) - ) + for i in range(x): + names.append("{0} {1}".format(choice(first), choice(second))) return set(names) first_names = ["Drew", "Mike", "Landon", "Jeremy", "Tyler", "Tom", "Avery"] last_names = ["Smith", "Jones", "Brighton", "Taylor"] names = random_name_generator(first_names, last_names, 5) -print '\n'.join(names) +print('\n'.join(names)) diff --git a/14_html_to_markdown.sh b/scripts/14_html_to_markdown.sh old mode 100644 new mode 100755 similarity index 100% rename from 14_html_to_markdown.sh rename to scripts/14_html_to_markdown.sh diff --git a/15_check_my_environment.py b/scripts/15_check_my_environment.py old mode 100644 new mode 100755 similarity index 91% rename from 15_check_my_environment.py rename to scripts/15_check_my_environment.py index 11017c4..62e0b8d --- a/15_check_my_environment.py +++ b/scripts/15_check_my_environment.py @@ -11,7 +11,7 @@ def __init__(self, configFile): pass def process(self): - print "ok" + print("ok") if __name__ == "__main__": m = Main(some_script.CONFIGFILE) @@ -39,7 +39,7 @@ def get_config_file(): if CONFIGFILE is None: sys.exit("Configuration error! Unknown environment set. \ Edit config.py and set appropriate environment") -print "Config file: {}".format(CONFIGFILE) +print("Config file: {}".format(CONFIGFILE)) if not os.path.exists(CONFIGFILE): sys.exit("Configuration error! Config file does not exist") -print "Config ok ...." +print("Config ok ....") diff --git a/16_jinja_quick_load.py b/scripts/16_jinja_quick_load.py old mode 100644 new mode 100755 similarity index 100% rename from 16_jinja_quick_load.py rename to scripts/16_jinja_quick_load.py diff --git a/17_rewrite_git_history.md b/scripts/17_rewrite_git_history.md old mode 100644 new mode 100755 similarity index 100% rename from 17_rewrite_git_history.md rename to scripts/17_rewrite_git_history.md diff --git a/18_zipper.py b/scripts/18_zipper.py similarity index 92% rename from 18_zipper.py rename to scripts/18_zipper.py index a350a70..43c956d 100755 --- a/18_zipper.py +++ b/scripts/18_zipper.py @@ -3,7 +3,7 @@ from zipfile import ZipFile -#set file name and time of creation +# set file name and time of creation today = datetime.now() file_name = 'zipper_' + today.strftime('%Y.%m.%dh%H%M') + '.zip' dir_name = 'tmp/' # update path diff --git a/19_tsv-to-csv.py b/scripts/19_tsv-to-csv.py old mode 100644 new mode 100755 similarity index 100% rename from 19_tsv-to-csv.py rename to scripts/19_tsv-to-csv.py diff --git a/20_restore_file_from_git.py b/scripts/20_restore_file_from_git.py old mode 100644 new mode 100755 similarity index 80% rename from 20_restore_file_from_git.py rename to scripts/20_restore_file_from_git.py index f692d9d..b1f581b --- a/20_restore_file_from_git.py +++ b/scripts/20_restore_file_from_git.py @@ -1,9 +1,9 @@ from subprocess import check_output, call -file_name = str(raw_input('Enter the file name: ')) +file_name = str(input('Enter the file name: ')) commit = check_output(["git", "rev-list", "-n", "1", "HEAD", "--", file_name]) -print str(commit).rstrip() +print(str(commit).rstrip()) call(["git", "checkout", str(commit).rstrip()+"~1", file_name]) diff --git a/scripts/21_twitter_bot.py b/scripts/21_twitter_bot.py new file mode 100755 index 0000000..dc0327f --- /dev/null +++ b/scripts/21_twitter_bot.py @@ -0,0 +1,25 @@ +import tweepy + +# Authentication credentials - dev.twitter.com +cfg = { + 'consumer_key': 'VALUE', + 'consumer_secret': 'VALUE', + 'access_token': 'VALUE', + 'access_token_secret': 'VALUE' +} + + +def get_api_handler(cfg): + auth = tweepy.OAuthHandler(cfg['consumer_key'], cfg['consumer_secret']) + auth.set_access_token(cfg['access_token'], cfg['access_token_secret']) + return tweepy.API(auth) + + +def main(): + api = get_api_handler(cfg) + tweet = 'Hello, world from Tweepy!' + api.update_status(status=tweet) + + +if __name__ == "__main__": + main() diff --git a/scripts/22_git_tag.py b/scripts/22_git_tag.py new file mode 100755 index 0000000..4849c07 --- /dev/null +++ b/scripts/22_git_tag.py @@ -0,0 +1,14 @@ +import subprocess +import sys + + +if len(sys.argv) == 3: + tag = sys.argv[1] + commit = sys.argv[2] + command = 'git tag -a {0} {1} -m "{2}"'.format(tag, commit, tag) + output = subprocess.check_output(command, shell=True).decode('utf-8') + subprocess.call(command, shell=True) + subprocess.call('git push --tags', shell=True) +else: + print('usage: tag.py TAG_NAME COMMIT') + sys.exit(1) diff --git a/scripts/23_flask_session_test.py b/scripts/23_flask_session_test.py new file mode 100755 index 0000000..9142760 --- /dev/null +++ b/scripts/23_flask_session_test.py @@ -0,0 +1,21 @@ +import sys +from flask import Flask, session, url_for, redirect + +app = Flask(__name__) +app.secret_key = 'secret' + + +@app.route('/') +def set(): + session.clear() + session['works'] = True + return redirect(url_for('get')) + + +@app.route('/get') +def get(): + works = session.get('works', False) + return str(works) + + +app.run(sys.argv[1], use_reloader=False) diff --git a/scripts/24_sql2csv.py b/scripts/24_sql2csv.py new file mode 100755 index 0000000..4e8f484 --- /dev/null +++ b/scripts/24_sql2csv.py @@ -0,0 +1,17 @@ +import sys +import csv +import sqlite3 + +if len(sys.argv) < 3: + print("Use: {0} DATABASE_NAME TABLE_NAME".format(sys.argv[0])) + exit() + +conn = sqlite3.connect(sys.argv[1]) +cur = conn.cursor() +data = cur.execute("SELECT * FROM {0}".format(sys.argv[2])) + +with open('output.csv', 'wb') as f: + writer = csv.writer(f) + writer.writerows(data) + +conn.close() diff --git a/scripts/25_ip2geolocation.py b/scripts/25_ip2geolocation.py new file mode 100755 index 0000000..f593676 --- /dev/null +++ b/scripts/25_ip2geolocation.py @@ -0,0 +1,61 @@ +import csv +import requests + + +def get_addresses(filename): + """ + Given a CSV file, this function returns a list of lists + where each element (list) in the outer list contains the + row info from the csv file. + """ + all_addresses = [] + with open(filename, 'rt') as f: + reader = csv.reader(f) + for row in reader: + all_addresses.append(row) + return all_addresses + + +def get_geolocation(all_the_ip_address): + """ + Given a list of lists from `get_addresses()`, this function + returns an updated lists of lists containing the geolocation. + """ + print("Getting geo information...") + updated_addresses = [] + counter = 1 + # update header + header_row = all_the_ip_address.pop(0) + header_row.extend(['Country', 'City']) + # get geolocation + for line in all_the_ip_address: + print("Grabbing geo info for row # {0}".format(counter)) + r = requests.get('https://freegeoip.net/json/%7B0%7D'.format(line[0])) + line.extend([str(r.json()['country_name']), str(r.json()['city'])]) + updated_addresses.append(line) + counter += 1 + updated_addresses.insert(0, header_row) + return updated_addresses + + +def create_csv(updated_address_list): + """ + Given the updated lists of lists from `get_geolocation()`, this function + creates a new CSV. + """ + import sys + if sys.version_info >= (3, 0, 0): + f = open('output.csv', 'w', newline='') + else: + f = open('output.csv', 'wb') + with f: + writer = csv.writer(f) + writer.writerows(updated_address_list) + print("All done!") + + +if __name__ == '__main__': + csv_file = '25_sample_csv.csv' + all_the_ip_address = get_addresses(csv_file) + updated_address_list = get_geolocation(all_the_ip_address) + create_csv(updated_address_list) diff --git a/scripts/25_sample_csv.csv b/scripts/25_sample_csv.csv new file mode 100755 index 0000000..9943c95 --- /dev/null +++ b/scripts/25_sample_csv.csv @@ -0,0 +1,6 @@ +IP Address,Full Name,Id,Email +162.252.85.172,Virgie Simonis,0,Tatyana_Barton@domenico.net +208.110.83.202,Tyrese Bartoletti,1,Birdie.Greenholt@annetta.co.uk +108.162.199.95,Markus Sanford,2,Lela_Homenick@philip.net +169.228.182.227,Anastasia Sawayn,3,Abe@camylle.name +184.72.242.188,Ashly Howe,5,Kieran.Bashirian@ansley.com \ No newline at end of file diff --git a/scripts/26_stock_scraper.py b/scripts/26_stock_scraper.py new file mode 100755 index 0000000..3e69cc2 --- /dev/null +++ b/scripts/26_stock_scraper.py @@ -0,0 +1,32 @@ +import requests +from lxml import html +from collections import defaultdict + + +def get_stocks(url): + # Make Request + page = requests.get(url) + # Parse/Scrape + tree = html.fromstring(page.text) + xpath = '//*[@id="mw-content-text"]/table[1]' + rows = tree.xpath(xpath)[0].findall("tr") + rows = [(row.getchildren()[0], row.getchildren()[3]) for row in rows[1:]] + rows = [(row[0].getchildren()[0].text, row[1].text) for row in rows] + industries = defaultdict(list) + for row in rows: + industries[row[1]].append(row[0]) + return industries + + +def output_data(data_dict): + for industry in data_dict: + print('\n'+industry) + print('-'*len(industry)) + for ticker in data_dict[industry]: + print(ticker) + + +if __name__ == '__main__': + url = 'http://en.wikipedia.org/wiki/List_of_S%26P_500_companies' + scraped_data = get_stocks(url) + output_data(scraped_data) diff --git a/scripts/27_send_sms.py b/scripts/27_send_sms.py new file mode 100755 index 0000000..0ad1d58 --- /dev/null +++ b/scripts/27_send_sms.py @@ -0,0 +1,12 @@ +import requests + +message = raw_input('Enter a Message: ') +number = raw_input('Enter the phone number: ') + + +payload = {'number': number, 'message': message} +r = requests.post("http://textbelt.com/text", data=payload) +if r.json()['success']: + print('Success!') +else: + print('Error!') diff --git a/scripts/28_income_tax_calculator.py b/scripts/28_income_tax_calculator.py new file mode 100755 index 0000000..3fc0758 --- /dev/null +++ b/scripts/28_income_tax_calculator.py @@ -0,0 +1,23 @@ +import requests + +headers = { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Accept': 'application/json', +} + +data = { + 'pay_rate': '10000', + 'filing_status': 'single', + 'pay_periods': 1, + 'state': 'CO', + 'year': + '2014' +} + +r = requests.post( + 'http://taxee.io/api/v1/calculate/2014', + data=data, + headers=headers +) + +print(r.text) diff --git a/scripts/29_json_test.json b/scripts/29_json_test.json new file mode 100755 index 0000000..ebe6459 --- /dev/null +++ b/scripts/29_json_test.json @@ -0,0 +1,31 @@ +{ + "colorsArray":[{ + "colorName":"red", + "hexValue":"#f00" + }, + { + "colorName":"green", + "hexValue":"#0f0" + }, + { + "colorName":"blue", + "hexValue":"#00f" + }, + { + "colorName":"cyan", + "hexValue":"#0ff" + }, + { + "colorName":"magenta", + "hexValue":"#f0f" + }, + { + "colorName":"yellow", + "hexValue":"#ff0" + }, + { + "colorName":"black", + "hexValue":"#000" + } + ] +} diff --git a/scripts/29_json_to_yaml.py b/scripts/29_json_to_yaml.py new file mode 100755 index 0000000..b22f64d --- /dev/null +++ b/scripts/29_json_to_yaml.py @@ -0,0 +1,16 @@ +import sys +import json +import yaml + +""" +Example usage: + +$ python 29_json_to_yaml.py 29_json_test.json +""" + +# load json data +json_data = json.loads(open(sys.argv[1]).read()) +# convert unicode to string +converted_json_data = json.dumps(json_data) +# output yaml +print(yaml.dump(yaml.load(converted_json_data), default_flow_style=False)) diff --git a/scripts/30_fullcontact.py b/scripts/30_fullcontact.py new file mode 100644 index 0000000..3ee2822 --- /dev/null +++ b/scripts/30_fullcontact.py @@ -0,0 +1,50 @@ +import os +import sys +import requests + +""" + +1. pip install requests +2. Obtain an API key: https://www.fullcontact.com/developer/pricing/ + +Example usage: + +$ python 30_fullcontact.py email SOME@EMAIL.COM +$ python 30_fullcontact.py twitter TWITTER_HANDLE +""" + + +# constants + +API_KEY = os.environ.get('FULLCONTACT_API_KEY') +BASE_URL = 'http://api.fullcontact.com/v2/person.json' + + +# helpers + +def get_arguments(): + if len(sys.argv) is 3: + return { + 'media': sys.argv[1], + 'user_info': sys.argv[2] + } + else: + print('Specify at least 1 argument') + sys.exit() + + +def call_api(contact): + url = BASE_URL + '?{0}={1}&apiKey={2}'.format( + contact['media'], contact['user_info'], API_KEY) + r = requests.get(url) + if r.status_code == 200: + return r.text + else: + return "Sorry, no results found." + + +# main + +if __name__ == "__main__": + media = get_arguments() + print(call_api(media)) diff --git a/scripts/31_youtube_sentiment.py b/scripts/31_youtube_sentiment.py new file mode 100644 index 0000000..f0f2129 --- /dev/null +++ b/scripts/31_youtube_sentiment.py @@ -0,0 +1,77 @@ +import sys +import requests +from bs4 import BeautifulSoup as bs4 + +""" +Example usage: + +$ python 31_youtube_sentiment.py https://www.youtube.com/watch?v=_vrAjAHhUsA +""" + + +def get_arguments(): + if len(sys.argv) is 2: + return sys.argv[1] + else: + print('Specify at least 1 argument') + sys.exit() + + +def get_comments(url): + html = requests.get('https://plus.googleapis.com/u/0/_/widget/render/comments?first_party_property=YOUTUBE&href=' + url) + soup = bs4(html.text, 'html.parser') + return [comment.string for comment in soup.findAll('div', class_='Ct')] + + +def calculate_sentiment(comments): + positive = 0 + negative = 0 + negative_words = [ + 'hate', 'hated', 'dislike', 'disliked', 'awful', 'terrible', 'bad', + 'painful', 'worst', 'suck', 'rubbish', 'sad', 'sodding' + ] + positive_words = [ + 'love', 'loved', 'like', 'liked', 'awesome', 'amazing', 'good', + 'great', 'excellent', 'brilliant', 'cool' + ] + for comment in comments: + if comment is None: + continue + else: + for word in comment.split(' '): + if word in negative_words: + negative += 1 + if word in positive_words: + positive += 1 + return {'positive': positive, 'negative': negative} + + +def main(): + url = get_arguments() + if url: + comments = get_comments(url) + if len(comments) <= 0: + print('This video has no comments.') + sys.exit() + sentiment = calculate_sentiment(comments) + positive_score = sentiment['positive'] + negative_score = sentiment['negative'] + total_score = positive_score + negative_score + if positive_score > negative_score: + print('This video is generally positive:') + print('{0} positive / {1} total hits'.format( + positive_score, total_score)) + elif negative_score > positive_score: + print('This video is generally negative:') + print ('{0} negative / {1} total hits'.format( + negative_score, total_score)) + else: + print('This video is mutual:') + print('{0} positive {1} negative'.format( + positive_score, negative_score)) + else: + print('No url supplied') + + +if __name__ == '__main__': + main() diff --git a/scripts/32_stock_scraper.py b/scripts/32_stock_scraper.py new file mode 100644 index 0000000..3bc2e7d --- /dev/null +++ b/scripts/32_stock_scraper.py @@ -0,0 +1,38 @@ +import urllib.request +from bs4 import BeautifulSoup + + +def get_stock_tickers(): + req = urllib.request.Request( + 'http://en.wikipedia.org/wiki/List_of_S%26P_500_companies') + page = urllib.request.urlopen(req) + soup = BeautifulSoup(page, 'html.parser') + table = soup.find('table', {'class': 'wikitable sortable'}) + tickers = [] + for row in table.findAll('tr'): + col = row.findAll('td') + if len(col) > 0: + tickers.append(str(col[0].string.strip())) + tickers.sort() + return tickers + + +def get_stock_prices(ticker_list): + for ticker in ticker_list: + htmlfile = urllib.request.urlopen( + "http://finance.yahoo.com/q?s={0}".format(ticker) + ) + htmltext = htmlfile.read() + soup = BeautifulSoup(htmltext, 'html.parser') + htmlSelector = 'yfs_l84_{0}'.format(ticker.lower()) + for price in soup.find_all(id=htmlSelector): + print('{0} is {1}'.format(ticker, price.text)) + + +def main(): + all_tickers = get_stock_tickers() + get_stock_prices(all_tickers) + + +if __name__ == '__main__': + main() diff --git a/scripts/33_country_code.py b/scripts/33_country_code.py new file mode 100644 index 0000000..134236c --- /dev/null +++ b/scripts/33_country_code.py @@ -0,0 +1,51 @@ +import csv +import sys +import json + +""" +Example usage: + +$ python 33_country_code.py 33_sample_csv.csv 33_country_codes.json +""" + + +def get_data(csv_file, json_file): + countryCodes = [] + countryNames = [] + continentNames = [] + with open(csv_file, 'rt') as file_one: + reader = csv.reader(file_one) + with open(json_file) as file_two: + json_data = json.load(file_two) + all_countries = json_data["country"] + for csv_row in reader: + for json_row in all_countries: + if csv_row[0] == json_row["countryCode"]: + countryCodes.append(json_row["countryCode"]) + countryNames.append(json_row["countryName"]) + continentNames.append(json_row["continentName"]) + + return [ + countryCodes, + countryNames, + continentNames + ] + + +def write_data(array_of_arrays): + with open('data.csv', 'wt') as csv_out: + writer = csv.writer(csv_out) + rows = zip( + array_of_arrays[0], + array_of_arrays[1], + array_of_arrays[2] + ) + for row in rows: + writer.writerow(row) + + +if __name__ == '__main__': + csv_file_name = sys.argv[1] + json_file_name = sys.argv[2] + data = get_data(csv_file_name, json_file_name) + write_data(data) diff --git a/scripts/33_country_codes.json b/scripts/33_country_codes.json new file mode 100644 index 0000000..4ec79e6 --- /dev/null +++ b/scripts/33_country_codes.json @@ -0,0 +1,1254 @@ +{ + "country": [ + { + "countryCode": "AD", + "countryName": "Andorra", + "continentName": "Europe" + }, + { + "countryCode": "AE", + "countryName": "United Arab Emirates", + "continentName": "Asia" + }, + { + "countryCode": "AF", + "countryName": "Afghanistan", + "continentName": "Asia" + }, + { + "countryCode": "AG", + "countryName": "Antigua and Barbuda", + "continentName": "North America" + }, + { + "countryCode": "AI", + "countryName": "Anguilla", + "continentName": "North America" + }, + { + "countryCode": "AL", + "countryName": "Albania", + "continentName": "Europe" + }, + { + "countryCode": "AM", + "countryName": "Armenia", + "continentName": "Asia" + }, + { + "countryCode": "AO", + "countryName": "Angola", + "continentName": "Africa" + }, + { + "countryCode": "AQ", + "countryName": "Antarctica", + "continentName": "Antarctica" + }, + { + "countryCode": "AR", + "countryName": "Argentina", + "continentName": "South America" + }, + { + "countryCode": "AS", + "countryName": "American Samoa", + "continentName": "Oceania" + }, + { + "countryCode": "AT", + "countryName": "Austria", + "continentName": "Europe" + }, + { + "countryCode": "AU", + "countryName": "Australia", + "continentName": "Oceania" + }, + { + "countryCode": "AW", + "countryName": "Aruba", + "continentName": "North America" + }, + { + "countryCode": "AX", + "countryName": "Åland", + "continentName": "Europe" + }, + { + "countryCode": "AZ", + "countryName": "Azerbaijan", + "continentName": "Asia" + }, + { + "countryCode": "BA", + "countryName": "Bosnia and Herzegovina", + "continentName": "Europe" + }, + { + "countryCode": "BB", + "countryName": "Barbados", + "continentName": "North America" + }, + { + "countryCode": "BD", + "countryName": "Bangladesh", + "continentName": "Asia" + }, + { + "countryCode": "BE", + "countryName": "Belgium", + "continentName": "Europe" + }, + { + "countryCode": "BF", + "countryName": "Burkina Faso", + "continentName": "Africa" + }, + { + "countryCode": "BG", + "countryName": "Bulgaria", + "continentName": "Europe" + }, + { + "countryCode": "BH", + "countryName": "Bahrain", + "continentName": "Asia" + }, + { + "countryCode": "BI", + "countryName": "Burundi", + "continentName": "Africa" + }, + { + "countryCode": "BJ", + "countryName": "Benin", + "continentName": "Africa" + }, + { + "countryCode": "BL", + "countryName": "Saint Barthélemy", + "continentName": "North America" + }, + { + "countryCode": "BM", + "countryName": "Bermuda", + "continentName": "North America" + }, + { + "countryCode": "BN", + "countryName": "Brunei", + "continentName": "Asia" + }, + { + "countryCode": "BO", + "countryName": "Bolivia", + "continentName": "South America" + }, + { + "countryCode": "BQ", + "countryName": "Bonaire", + "continentName": "North America" + }, + { + "countryCode": "BR", + "countryName": "Brazil", + "continentName": "South America" + }, + { + "countryCode": "BS", + "countryName": "Bahamas", + "continentName": "North America" + }, + { + "countryCode": "BT", + "countryName": "Bhutan", + "continentName": "Asia" + }, + { + "countryCode": "BV", + "countryName": "Bouvet Island", + "continentName": "Antarctica" + }, + { + "countryCode": "BW", + "countryName": "Botswana", + "continentName": "Africa" + }, + { + "countryCode": "BY", + "countryName": "Belarus", + "continentName": "Europe" + }, + { + "countryCode": "BZ", + "countryName": "Belize", + "continentName": "North America" + }, + { + "countryCode": "CA", + "countryName": "Canada", + "continentName": "North America" + }, + { + "countryCode": "CC", + "countryName": "Cocos [Keeling] Islands", + "continentName": "Asia" + }, + { + "countryCode": "CD", + "countryName": "Democratic Republic of the Congo", + "continentName": "Africa" + }, + { + "countryCode": "CF", + "countryName": "Central African Republic", + "continentName": "Africa" + }, + { + "countryCode": "CG", + "countryName": "Republic of the Congo", + "continentName": "Africa" + }, + { + "countryCode": "CH", + "countryName": "Switzerland", + "continentName": "Europe" + }, + { + "countryCode": "CI", + "countryName": "Ivory Coast", + "continentName": "Africa" + }, + { + "countryCode": "CK", + "countryName": "Cook Islands", + "continentName": "Oceania" + }, + { + "countryCode": "CL", + "countryName": "Chile", + "continentName": "South America" + }, + { + "countryCode": "CM", + "countryName": "Cameroon", + "continentName": "Africa" + }, + { + "countryCode": "CN", + "countryName": "China", + "continentName": "Asia" + }, + { + "countryCode": "CO", + "countryName": "Colombia", + "continentName": "South America" + }, + { + "countryCode": "CR", + "countryName": "Costa Rica", + "continentName": "North America" + }, + { + "countryCode": "CU", + "countryName": "Cuba", + "continentName": "North America" + }, + { + "countryCode": "CV", + "countryName": "Cape Verde", + "continentName": "Africa" + }, + { + "countryCode": "CW", + "countryName": "Curacao", + "continentName": "North America" + }, + { + "countryCode": "CX", + "countryName": "Christmas Island", + "continentName": "Asia" + }, + { + "countryCode": "CY", + "countryName": "Cyprus", + "continentName": "Europe" + }, + { + "countryCode": "CZ", + "countryName": "Czechia", + "continentName": "Europe" + }, + { + "countryCode": "DE", + "countryName": "Germany", + "continentName": "Europe" + }, + { + "countryCode": "DJ", + "countryName": "Djibouti", + "continentName": "Africa" + }, + { + "countryCode": "DK", + "countryName": "Denmark", + "continentName": "Europe" + }, + { + "countryCode": "DM", + "countryName": "Dominica", + "continentName": "North America" + }, + { + "countryCode": "DO", + "countryName": "Dominican Republic", + "continentName": "North America" + }, + { + "countryCode": "DZ", + "countryName": "Algeria", + "continentName": "Africa" + }, + { + "countryCode": "EC", + "countryName": "Ecuador", + "continentName": "South America" + }, + { + "countryCode": "EE", + "countryName": "Estonia", + "continentName": "Europe" + }, + { + "countryCode": "EG", + "countryName": "Egypt", + "continentName": "Africa" + }, + { + "countryCode": "EH", + "countryName": "Western Sahara", + "continentName": "Africa" + }, + { + "countryCode": "ER", + "countryName": "Eritrea", + "continentName": "Africa" + }, + { + "countryCode": "ES", + "countryName": "Spain", + "continentName": "Europe" + }, + { + "countryCode": "ET", + "countryName": "Ethiopia", + "continentName": "Africa" + }, + { + "countryCode": "FI", + "countryName": "Finland", + "continentName": "Europe" + }, + { + "countryCode": "FJ", + "countryName": "Fiji", + "continentName": "Oceania" + }, + { + "countryCode": "FK", + "countryName": "Falkland Islands", + "continentName": "South America" + }, + { + "countryCode": "FM", + "countryName": "Micronesia", + "continentName": "Oceania" + }, + { + "countryCode": "FO", + "countryName": "Faroe Islands", + "continentName": "Europe" + }, + { + "countryCode": "FR", + "countryName": "France", + "continentName": "Europe" + }, + { + "countryCode": "GA", + "countryName": "Gabon", + "continentName": "Africa" + }, + { + "countryCode": "GB", + "countryName": "United Kingdom", + "continentName": "Europe" + }, + { + "countryCode": "GD", + "countryName": "Grenada", + "continentName": "North America" + }, + { + "countryCode": "GE", + "countryName": "Georgia", + "continentName": "Asia" + }, + { + "countryCode": "GF", + "countryName": "French Guiana", + "continentName": "South America" + }, + { + "countryCode": "GG", + "countryName": "Guernsey", + "continentName": "Europe" + }, + { + "countryCode": "GH", + "countryName": "Ghana", + "continentName": "Africa" + }, + { + "countryCode": "GI", + "countryName": "Gibraltar", + "continentName": "Europe" + }, + { + "countryCode": "GL", + "countryName": "Greenland", + "continentName": "North America" + }, + { + "countryCode": "GM", + "countryName": "Gambia", + "continentName": "Africa" + }, + { + "countryCode": "GN", + "countryName": "Guinea", + "continentName": "Africa" + }, + { + "countryCode": "GP", + "countryName": "Guadeloupe", + "continentName": "North America" + }, + { + "countryCode": "GQ", + "countryName": "Equatorial Guinea", + "continentName": "Africa" + }, + { + "countryCode": "GR", + "countryName": "Greece", + "continentName": "Europe" + }, + { + "countryCode": "GS", + "countryName": "South Georgia and the South Sandwich Islands", + "continentName": "Antarctica" + }, + { + "countryCode": "GT", + "countryName": "Guatemala", + "continentName": "North America" + }, + { + "countryCode": "GU", + "countryName": "Guam", + "continentName": "Oceania" + }, + { + "countryCode": "GW", + "countryName": "Guinea-Bissau", + "continentName": "Africa" + }, + { + "countryCode": "GY", + "countryName": "Guyana", + "continentName": "South America" + }, + { + "countryCode": "HK", + "countryName": "Hong Kong", + "continentName": "Asia" + }, + { + "countryCode": "HM", + "countryName": "Heard Island and McDonald Islands", + "continentName": "Antarctica" + }, + { + "countryCode": "HN", + "countryName": "Honduras", + "continentName": "North America" + }, + { + "countryCode": "HR", + "countryName": "Croatia", + "continentName": "Europe" + }, + { + "countryCode": "HT", + "countryName": "Haiti", + "continentName": "North America" + }, + { + "countryCode": "HU", + "countryName": "Hungary", + "continentName": "Europe" + }, + { + "countryCode": "ID", + "countryName": "Indonesia", + "continentName": "Asia" + }, + { + "countryCode": "IE", + "countryName": "Ireland", + "continentName": "Europe" + }, + { + "countryCode": "IL", + "countryName": "Israel", + "continentName": "Asia" + }, + { + "countryCode": "IM", + "countryName": "Isle of Man", + "continentName": "Europe" + }, + { + "countryCode": "IN", + "countryName": "India", + "continentName": "Asia" + }, + { + "countryCode": "IO", + "countryName": "British Indian Ocean Territory", + "continentName": "Asia" + }, + { + "countryCode": "IQ", + "countryName": "Iraq", + "continentName": "Asia" + }, + { + "countryCode": "IR", + "countryName": "Iran", + "continentName": "Asia" + }, + { + "countryCode": "IS", + "countryName": "Iceland", + "continentName": "Europe" + }, + { + "countryCode": "IT", + "countryName": "Italy", + "continentName": "Europe" + }, + { + "countryCode": "JE", + "countryName": "Jersey", + "continentName": "Europe" + }, + { + "countryCode": "JM", + "countryName": "Jamaica", + "continentName": "North America" + }, + { + "countryCode": "JO", + "countryName": "Jordan", + "continentName": "Asia" + }, + { + "countryCode": "JP", + "countryName": "Japan", + "continentName": "Asia" + }, + { + "countryCode": "KE", + "countryName": "Kenya", + "continentName": "Africa" + }, + { + "countryCode": "KG", + "countryName": "Kyrgyzstan", + "continentName": "Asia" + }, + { + "countryCode": "KH", + "countryName": "Cambodia", + "continentName": "Asia" + }, + { + "countryCode": "KI", + "countryName": "Kiribati", + "continentName": "Oceania" + }, + { + "countryCode": "KM", + "countryName": "Comoros", + "continentName": "Africa" + }, + { + "countryCode": "KN", + "countryName": "Saint Kitts and Nevis", + "continentName": "North America" + }, + { + "countryCode": "KP", + "countryName": "North Korea", + "continentName": "Asia" + }, + { + "countryCode": "KR", + "countryName": "South Korea", + "continentName": "Asia" + }, + { + "countryCode": "KW", + "countryName": "Kuwait", + "continentName": "Asia" + }, + { + "countryCode": "KY", + "countryName": "Cayman Islands", + "continentName": "North America" + }, + { + "countryCode": "KZ", + "countryName": "Kazakhstan", + "continentName": "Asia" + }, + { + "countryCode": "LA", + "countryName": "Laos", + "continentName": "Asia" + }, + { + "countryCode": "LB", + "countryName": "Lebanon", + "continentName": "Asia" + }, + { + "countryCode": "LC", + "countryName": "Saint Lucia", + "continentName": "North America" + }, + { + "countryCode": "LI", + "countryName": "Liechtenstein", + "continentName": "Europe" + }, + { + "countryCode": "LK", + "countryName": "Sri Lanka", + "continentName": "Asia" + }, + { + "countryCode": "LR", + "countryName": "Liberia", + "continentName": "Africa" + }, + { + "countryCode": "LS", + "countryName": "Lesotho", + "continentName": "Africa" + }, + { + "countryCode": "LT", + "countryName": "Lithuania", + "continentName": "Europe" + }, + { + "countryCode": "LU", + "countryName": "Luxembourg", + "continentName": "Europe" + }, + { + "countryCode": "LV", + "countryName": "Latvia", + "continentName": "Europe" + }, + { + "countryCode": "LY", + "countryName": "Libya", + "continentName": "Africa" + }, + { + "countryCode": "MA", + "countryName": "Morocco", + "continentName": "Africa" + }, + { + "countryCode": "MC", + "countryName": "Monaco", + "continentName": "Europe" + }, + { + "countryCode": "MD", + "countryName": "Moldova", + "continentName": "Europe" + }, + { + "countryCode": "ME", + "countryName": "Montenegro", + "continentName": "Europe" + }, + { + "countryCode": "MF", + "countryName": "Saint Martin", + "continentName": "North America" + }, + { + "countryCode": "MG", + "countryName": "Madagascar", + "continentName": "Africa" + }, + { + "countryCode": "MH", + "countryName": "Marshall Islands", + "continentName": "Oceania" + }, + { + "countryCode": "MK", + "countryName": "Macedonia", + "continentName": "Europe" + }, + { + "countryCode": "ML", + "countryName": "Mali", + "continentName": "Africa" + }, + { + "countryCode": "MM", + "countryName": "Myanmar [Burma]", + "continentName": "Asia" + }, + { + "countryCode": "MN", + "countryName": "Mongolia", + "continentName": "Asia" + }, + { + "countryCode": "MO", + "countryName": "Macao", + "continentName": "Asia" + }, + { + "countryCode": "MP", + "countryName": "Northern Mariana Islands", + "continentName": "Oceania" + }, + { + "countryCode": "MQ", + "countryName": "Martinique", + "continentName": "North America" + }, + { + "countryCode": "MR", + "countryName": "Mauritania", + "continentName": "Africa" + }, + { + "countryCode": "MS", + "countryName": "Montserrat", + "continentName": "North America" + }, + { + "countryCode": "MT", + "countryName": "Malta", + "continentName": "Europe" + }, + { + "countryCode": "MU", + "countryName": "Mauritius", + "continentName": "Africa" + }, + { + "countryCode": "MV", + "countryName": "Maldives", + "continentName": "Asia" + }, + { + "countryCode": "MW", + "countryName": "Malawi", + "continentName": "Africa" + }, + { + "countryCode": "MX", + "countryName": "Mexico", + "continentName": "North America" + }, + { + "countryCode": "MY", + "countryName": "Malaysia", + "continentName": "Asia" + }, + { + "countryCode": "MZ", + "countryName": "Mozambique", + "continentName": "Africa" + }, + { + "countryCode": "NA", + "countryName": "Namibia", + "continentName": "Africa" + }, + { + "countryCode": "NC", + "countryName": "New Caledonia", + "continentName": "Oceania" + }, + { + "countryCode": "NE", + "countryName": "Niger", + "continentName": "Africa" + }, + { + "countryCode": "NF", + "countryName": "Norfolk Island", + "continentName": "Oceania" + }, + { + "countryCode": "NG", + "countryName": "Nigeria", + "continentName": "Africa" + }, + { + "countryCode": "NI", + "countryName": "Nicaragua", + "continentName": "North America" + }, + { + "countryCode": "NL", + "countryName": "Netherlands", + "continentName": "Europe" + }, + { + "countryCode": "NO", + "countryName": "Norway", + "continentName": "Europe" + }, + { + "countryCode": "NP", + "countryName": "Nepal", + "continentName": "Asia" + }, + { + "countryCode": "NR", + "countryName": "Nauru", + "continentName": "Oceania" + }, + { + "countryCode": "NU", + "countryName": "Niue", + "continentName": "Oceania" + }, + { + "countryCode": "NZ", + "countryName": "New Zealand", + "continentName": "Oceania" + }, + { + "countryCode": "OM", + "countryName": "Oman", + "continentName": "Asia" + }, + { + "countryCode": "PA", + "countryName": "Panama", + "continentName": "North America" + }, + { + "countryCode": "PE", + "countryName": "Peru", + "continentName": "South America" + }, + { + "countryCode": "PF", + "countryName": "French Polynesia", + "continentName": "Oceania" + }, + { + "countryCode": "PG", + "countryName": "Papua New Guinea", + "continentName": "Oceania" + }, + { + "countryCode": "PH", + "countryName": "Philippines", + "continentName": "Asia" + }, + { + "countryCode": "PK", + "countryName": "Pakistan", + "continentName": "Asia" + }, + { + "countryCode": "PL", + "countryName": "Poland", + "continentName": "Europe" + }, + { + "countryCode": "PM", + "countryName": "Saint Pierre and Miquelon", + "continentName": "North America" + }, + { + "countryCode": "PN", + "countryName": "Pitcairn Islands", + "continentName": "Oceania" + }, + { + "countryCode": "PR", + "countryName": "Puerto Rico", + "continentName": "North America" + }, + { + "countryCode": "PS", + "countryName": "Palestine", + "continentName": "Asia" + }, + { + "countryCode": "PT", + "countryName": "Portugal", + "continentName": "Europe" + }, + { + "countryCode": "PW", + "countryName": "Palau", + "continentName": "Oceania" + }, + { + "countryCode": "PY", + "countryName": "Paraguay", + "continentName": "South America" + }, + { + "countryCode": "QA", + "countryName": "Qatar", + "continentName": "Asia" + }, + { + "countryCode": "RE", + "countryName": "Réunion", + "continentName": "Africa" + }, + { + "countryCode": "RO", + "countryName": "Romania", + "continentName": "Europe" + }, + { + "countryCode": "RS", + "countryName": "Serbia", + "continentName": "Europe" + }, + { + "countryCode": "RU", + "countryName": "Russia", + "continentName": "Europe" + }, + { + "countryCode": "RW", + "countryName": "Rwanda", + "continentName": "Africa" + }, + { + "countryCode": "SA", + "countryName": "Saudi Arabia", + "continentName": "Asia" + }, + { + "countryCode": "SB", + "countryName": "Solomon Islands", + "continentName": "Oceania" + }, + { + "countryCode": "SC", + "countryName": "Seychelles", + "continentName": "Africa" + }, + { + "countryCode": "SD", + "countryName": "Sudan", + "continentName": "Africa" + }, + { + "countryCode": "SE", + "countryName": "Sweden", + "continentName": "Europe" + }, + { + "countryCode": "SG", + "countryName": "Singapore", + "continentName": "Asia" + }, + { + "countryCode": "SH", + "countryName": "Saint Helena", + "continentName": "Africa" + }, + { + "countryCode": "SI", + "countryName": "Slovenia", + "continentName": "Europe" + }, + { + "countryCode": "SJ", + "countryName": "Svalbard and Jan Mayen", + "continentName": "Europe" + }, + { + "countryCode": "SK", + "countryName": "Slovakia", + "continentName": "Europe" + }, + { + "countryCode": "SL", + "countryName": "Sierra Leone", + "continentName": "Africa" + }, + { + "countryCode": "SM", + "countryName": "San Marino", + "continentName": "Europe" + }, + { + "countryCode": "SN", + "countryName": "Senegal", + "continentName": "Africa" + }, + { + "countryCode": "SO", + "countryName": "Somalia", + "continentName": "Africa" + }, + { + "countryCode": "SR", + "countryName": "Suriname", + "continentName": "South America" + }, + { + "countryCode": "SS", + "countryName": "South Sudan", + "continentName": "Africa" + }, + { + "countryCode": "ST", + "countryName": "São Tomé and Príncipe", + "continentName": "Africa" + }, + { + "countryCode": "SV", + "countryName": "El Salvador", + "continentName": "North America" + }, + { + "countryCode": "SX", + "countryName": "Sint Maarten", + "continentName": "North America" + }, + { + "countryCode": "SY", + "countryName": "Syria", + "continentName": "Asia" + }, + { + "countryCode": "SZ", + "countryName": "Swaziland", + "continentName": "Africa" + }, + { + "countryCode": "TC", + "countryName": "Turks and Caicos Islands", + "continentName": "North America" + }, + { + "countryCode": "TD", + "countryName": "Chad", + "continentName": "Africa" + }, + { + "countryCode": "TF", + "countryName": "French Southern Territories", + "continentName": "Antarctica" + }, + { + "countryCode": "TG", + "countryName": "Togo", + "continentName": "Africa" + }, + { + "countryCode": "TH", + "countryName": "Thailand", + "continentName": "Asia" + }, + { + "countryCode": "TJ", + "countryName": "Tajikistan", + "continentName": "Asia" + }, + { + "countryCode": "TK", + "countryName": "Tokelau", + "continentName": "Oceania" + }, + { + "countryCode": "TL", + "countryName": "East Timor", + "continentName": "Oceania" + }, + { + "countryCode": "TM", + "countryName": "Turkmenistan", + "continentName": "Asia" + }, + { + "countryCode": "TN", + "countryName": "Tunisia", + "continentName": "Africa" + }, + { + "countryCode": "TO", + "countryName": "Tonga", + "continentName": "Oceania" + }, + { + "countryCode": "TR", + "countryName": "Turkey", + "continentName": "Asia" + }, + { + "countryCode": "TT", + "countryName": "Trinidad and Tobago", + "continentName": "North America" + }, + { + "countryCode": "TV", + "countryName": "Tuvalu", + "continentName": "Oceania" + }, + { + "countryCode": "TW", + "countryName": "Taiwan", + "continentName": "Asia" + }, + { + "countryCode": "TZ", + "countryName": "Tanzania", + "continentName": "Africa" + }, + { + "countryCode": "UA", + "countryName": "Ukraine", + "continentName": "Europe" + }, + { + "countryCode": "UG", + "countryName": "Uganda", + "continentName": "Africa" + }, + { + "countryCode": "UM", + "countryName": "U.S. Minor Outlying Islands", + "continentName": "Oceania" + }, + { + "countryCode": "US", + "countryName": "United States", + "continentName": "North America" + }, + { + "countryCode": "UY", + "countryName": "Uruguay", + "continentName": "South America" + }, + { + "countryCode": "UZ", + "countryName": "Uzbekistan", + "continentName": "Asia" + }, + { + "countryCode": "VA", + "countryName": "Vatican City", + "continentName": "Europe" + }, + { + "countryCode": "VC", + "countryName": "Saint Vincent and the Grenadines", + "continentName": "North America" + }, + { + "countryCode": "VE", + "countryName": "Venezuela", + "continentName": "South America" + }, + { + "countryCode": "VG", + "countryName": "British Virgin Islands", + "continentName": "North America" + }, + { + "countryCode": "VI", + "countryName": "U.S. Virgin Islands", + "continentName": "North America" + }, + { + "countryCode": "VN", + "countryName": "Vietnam", + "continentName": "Asia" + }, + { + "countryCode": "VU", + "countryName": "Vanuatu", + "continentName": "Oceania" + }, + { + "countryCode": "WF", + "countryName": "Wallis and Futuna", + "continentName": "Oceania" + }, + { + "countryCode": "WS", + "countryName": "Samoa", + "continentName": "Oceania" + }, + { + "countryCode": "XK", + "countryName": "Kosovo", + "continentName": "Europe" + }, + { + "countryCode": "YE", + "countryName": "Yemen", + "continentName": "Asia" + }, + { + "countryCode": "YT", + "countryName": "Mayotte", + "continentName": "Africa" + }, + { + "countryCode": "ZA", + "countryName": "South Africa", + "continentName": "Africa" + }, + { + "countryCode": "ZM", + "countryName": "Zambia", + "continentName": "Africa" + }, + { + "countryCode": "ZW", + "countryName": "Zimbabwe", + "continentName": "Africa" + } + ] +} diff --git a/scripts/33_sample_csv.csv b/scripts/33_sample_csv.csv new file mode 100644 index 0000000..62b23a2 --- /dev/null +++ b/scripts/33_sample_csv.csv @@ -0,0 +1,109 @@ +A2 +AE +AL +AP +AR +AT +AU +AZ +BA +BD +BE +BG +BH +BN +BR +BY +CA +CH +CL +CN +CO +CR +CW +CY +CZ +DE +DK +DO +EC +EE +ES +FI +FR +GB +GE +GG +GH +GI +GR +GT +HK +HR +HT +HU +ID +IE +IL +IN +IS +IT +JM +JO +JP +KE +KG +KR +KW +KY +KZ +LA +LB +LK +LT +LU +LV +MD +MG +MK +MO +MT +MV +MX +MY +NC +NG +NI +NL +NO +NP +NZ +OM +PA +PE +PH +PK +PL +PR +PT +PY +RO +RS +RU +SA +SE +SG +SI +SK +SO +TH +TR +TW +TZ +UA +US +UY +VN +VU +ZA +ZW diff --git a/scripts/34_git_all_repos.py b/scripts/34_git_all_repos.py new file mode 100644 index 0000000..b3e2d5b --- /dev/null +++ b/scripts/34_git_all_repos.py @@ -0,0 +1,42 @@ +import sys +import os +import requests + + +def get_total_repos(group, name): + repo_urls = [] + page = 1 + while True: + url = 'https://api.github.com/%7B0%7D/%7B1%7D/repos?per_page=100&page={2}' + r = requests.get(url.format(group, name, page)) + if r.status_code == 200: + rdata = r.json() + for repo in rdata: + repo_urls.append(repo['clone_url']) + if (len(rdata) >= 100): + page += 1 + else: + print('Found {0} repos.'.format(len(repo_urls))) + break + else: + print(r) + return False + return repo_urls + + +def clone_repos(all_repos): + count = 1 + print('Cloning...') + for repo in all_repos: + os.system('Git clone ' + repo) + print('Completed repo #{0} of {1}'.format(count, len(all_repos))) + count += 1 + +if __name__ == '__main__': + if len(sys.argv) > 2: + total = get_total_repos(sys.argv[1], sys.argv[2]) + if total: + clone_repos(total) + + else: + print('Usage: python USERS_OR_ORG GITHUB_USER_OR_ORG-NAME') diff --git a/scripts/data.csv b/scripts/data.csv new file mode 100644 index 0000000..3f0c610 --- /dev/null +++ b/scripts/data.csv @@ -0,0 +1,107 @@ +AE,United Arab Emirates,Asia +AL,Albania,Europe +AR,Argentina,South America +AT,Austria,Europe +AU,Australia,Oceania +AZ,Azerbaijan,Asia +BA,Bosnia and Herzegovina,Europe +BD,Bangladesh,Asia +BE,Belgium,Europe +BG,Bulgaria,Europe +BH,Bahrain,Asia +BN,Brunei,Asia +BR,Brazil,South America +BY,Belarus,Europe +CA,Canada,North America +CH,Switzerland,Europe +CL,Chile,South America +CN,China,Asia +CO,Colombia,South America +CR,Costa Rica,North America +CW,Curacao,North America +CY,Cyprus,Europe +CZ,Czechia,Europe +DE,Germany,Europe +DK,Denmark,Europe +DO,Dominican Republic,North America +EC,Ecuador,South America +EE,Estonia,Europe +ES,Spain,Europe +FI,Finland,Europe +FR,France,Europe +GB,United Kingdom,Europe +GE,Georgia,Asia +GG,Guernsey,Europe +GH,Ghana,Africa +GI,Gibraltar,Europe +GR,Greece,Europe +GT,Guatemala,North America +HK,Hong Kong,Asia +HR,Croatia,Europe +HT,Haiti,North America +HU,Hungary,Europe +ID,Indonesia,Asia +IE,Ireland,Europe +IL,Israel,Asia +IN,India,Asia +IS,Iceland,Europe +IT,Italy,Europe +JM,Jamaica,North America +JO,Jordan,Asia +JP,Japan,Asia +KE,Kenya,Africa +KG,Kyrgyzstan,Asia +KR,South Korea,Asia +KW,Kuwait,Asia +KY,Cayman Islands,North America +KZ,Kazakhstan,Asia +LA,Laos,Asia +LB,Lebanon,Asia +LK,Sri Lanka,Asia +LT,Lithuania,Europe +LU,Luxembourg,Europe +LV,Latvia,Europe +MD,Moldova,Europe +MG,Madagascar,Africa +MK,Macedonia,Europe +MO,Macao,Asia +MT,Malta,Europe +MV,Maldives,Asia +MX,Mexico,North America +MY,Malaysia,Asia +NC,New Caledonia,Oceania +NG,Nigeria,Africa +NI,Nicaragua,North America +NL,Netherlands,Europe +NO,Norway,Europe +NP,Nepal,Asia +NZ,New Zealand,Oceania +OM,Oman,Asia +PA,Panama,North America +PE,Peru,South America +PH,Philippines,Asia +PK,Pakistan,Asia +PL,Poland,Europe +PR,Puerto Rico,North America +PT,Portugal,Europe +PY,Paraguay,South America +RO,Romania,Europe +RS,Serbia,Europe +RU,Russia,Europe +SA,Saudi Arabia,Asia +SE,Sweden,Europe +SG,Singapore,Asia +SI,Slovenia,Europe +SK,Slovakia,Europe +SO,Somalia,Africa +TH,Thailand,Asia +TR,Turkey,Asia +TW,Taiwan,Asia +TZ,Tanzania,Africa +UA,Ukraine,Europe +US,United States,North America +UY,Uruguay,South America +VN,Vietnam,Asia +VU,Vanuatu,Oceania +ZA,South Africa,Africa +ZW,Zimbabwe,Africa