From 4e54276ad46cd11a781b3525221ee719a0895a48 Mon Sep 17 00:00:00 2001 From: Michael Herman Date: Sat, 17 Dec 2016 09:22:43 -0700 Subject: [PATCH 1/9] updated folder structure --- 01_remove_all_pyc.md => scripts/01_remove_all_pyc.md | 0 02_find_all_links.py => scripts/02_find_all_links.py | 0 .../03_simple_twitter_manager.py | 0 04_rename_with_slice.py => scripts/04_rename_with_slice.py | 0 .../05_load_json_without_dupes.py | 0 06_execution_time.py => scripts/06_execution_time.py | 0 .../07_benchmark_permissions_loading_django.py | 0 .../08_basic_email_web_crawler.py | 0 .../09_basic_link_web_crawler.py | 0 .../10_find_files_recursively.py | 4 ++-- .../11_optimize_images_with_wand.py | 0 12_csv_split.py => scripts/12_csv_split.py | 0 12_sample_csv.csv => scripts/12_sample_csv.csv | 0 .../13_random_name_generator.py | 0 14_html_to_markdown.sh => scripts/14_html_to_markdown.sh | 0 .../15_check_my_environment.py | 0 16_jinja_quick_load.py => scripts/16_jinja_quick_load.py | 0 .../17_rewrite_git_history.md | 0 18_zipper.py => scripts/18_zipper.py | 0 19_tsv-to-csv.py => scripts/19_tsv-to-csv.py | 0 .../20_restore_file_from_git.py | 0 21_twitter_bot.py => scripts/21_twitter_bot.py | 0 22_git_tag.py => scripts/22_git_tag.py | 0 23_flask_session_test.py => scripts/23_flask_session_test.py | 0 24_sql2csv.py => scripts/24_sql2csv.py | 0 25_ip2geolocation.py => scripts/25_ip2geolocation.py | 0 25_sample_csv.csv => scripts/25_sample_csv.csv | 0 26_stock_scraper.py => scripts/26_stock_scraper.py | 0 27_send_sms.py => scripts/27_send_sms.py | 0 .../28_income_tax_calculator.py | 0 29_json_test.json => scripts/29_json_test.json | 0 29_json_to_yaml.py => scripts/29_json_to_yaml.py | 0 30_fullcontact.py => scripts/30_fullcontact.py | 0 31_youtube_sentiment.py => scripts/31_youtube_sentiment.py | 0 32_stock_scraper.py => scripts/32_stock_scraper.py | 0 33_country_code.py => scripts/33_country_code.py | 0 33_country_codes.json => scripts/33_country_codes.json | 0 33_sample_csv.csv => scripts/33_sample_csv.csv | 0 data.csv => scripts/data.csv | 0 39 files changed, 2 insertions(+), 2 deletions(-) rename 01_remove_all_pyc.md => scripts/01_remove_all_pyc.md (100%) rename 02_find_all_links.py => scripts/02_find_all_links.py (100%) rename 03_simple_twitter_manager.py => scripts/03_simple_twitter_manager.py (100%) rename 04_rename_with_slice.py => scripts/04_rename_with_slice.py (100%) rename 05_load_json_without_dupes.py => scripts/05_load_json_without_dupes.py (100%) rename 06_execution_time.py => scripts/06_execution_time.py (100%) rename 07_benchmark_permissions_loading_django.py => scripts/07_benchmark_permissions_loading_django.py (100%) rename 08_basic_email_web_crawler.py => scripts/08_basic_email_web_crawler.py (100%) rename 09_basic_link_web_crawler.py => scripts/09_basic_link_web_crawler.py (100%) rename 10_find_files_recursively.py => scripts/10_find_files_recursively.py (92%) rename 11_optimize_images_with_wand.py => scripts/11_optimize_images_with_wand.py (100%) rename 12_csv_split.py => scripts/12_csv_split.py (100%) rename 12_sample_csv.csv => scripts/12_sample_csv.csv (100%) rename 13_random_name_generator.py => scripts/13_random_name_generator.py (100%) rename 14_html_to_markdown.sh => scripts/14_html_to_markdown.sh (100%) rename 15_check_my_environment.py => scripts/15_check_my_environment.py (100%) rename 16_jinja_quick_load.py => scripts/16_jinja_quick_load.py (100%) rename 17_rewrite_git_history.md => scripts/17_rewrite_git_history.md (100%) rename 18_zipper.py => scripts/18_zipper.py (100%) rename 19_tsv-to-csv.py => scripts/19_tsv-to-csv.py (100%) rename 20_restore_file_from_git.py => scripts/20_restore_file_from_git.py (100%) rename 21_twitter_bot.py => scripts/21_twitter_bot.py (100%) rename 22_git_tag.py => scripts/22_git_tag.py (100%) rename 23_flask_session_test.py => scripts/23_flask_session_test.py (100%) rename 24_sql2csv.py => scripts/24_sql2csv.py (100%) rename 25_ip2geolocation.py => scripts/25_ip2geolocation.py (100%) rename 25_sample_csv.csv => scripts/25_sample_csv.csv (100%) rename 26_stock_scraper.py => scripts/26_stock_scraper.py (100%) rename 27_send_sms.py => scripts/27_send_sms.py (100%) rename 28_income_tax_calculator.py => scripts/28_income_tax_calculator.py (100%) rename 29_json_test.json => scripts/29_json_test.json (100%) rename 29_json_to_yaml.py => scripts/29_json_to_yaml.py (100%) rename 30_fullcontact.py => scripts/30_fullcontact.py (100%) rename 31_youtube_sentiment.py => scripts/31_youtube_sentiment.py (100%) rename 32_stock_scraper.py => scripts/32_stock_scraper.py (100%) rename 33_country_code.py => scripts/33_country_code.py (100%) rename 33_country_codes.json => scripts/33_country_codes.json (100%) rename 33_sample_csv.csv => scripts/33_sample_csv.csv (100%) rename data.csv => scripts/data.csv (100%) diff --git a/01_remove_all_pyc.md b/scripts/01_remove_all_pyc.md similarity index 100% rename from 01_remove_all_pyc.md rename to scripts/01_remove_all_pyc.md diff --git a/02_find_all_links.py b/scripts/02_find_all_links.py similarity index 100% rename from 02_find_all_links.py rename to scripts/02_find_all_links.py diff --git a/03_simple_twitter_manager.py b/scripts/03_simple_twitter_manager.py similarity index 100% rename from 03_simple_twitter_manager.py rename to scripts/03_simple_twitter_manager.py diff --git a/04_rename_with_slice.py b/scripts/04_rename_with_slice.py similarity index 100% rename from 04_rename_with_slice.py rename to scripts/04_rename_with_slice.py diff --git a/05_load_json_without_dupes.py b/scripts/05_load_json_without_dupes.py similarity index 100% rename from 05_load_json_without_dupes.py rename to scripts/05_load_json_without_dupes.py diff --git a/06_execution_time.py b/scripts/06_execution_time.py similarity index 100% rename from 06_execution_time.py rename to scripts/06_execution_time.py diff --git a/07_benchmark_permissions_loading_django.py b/scripts/07_benchmark_permissions_loading_django.py similarity index 100% rename from 07_benchmark_permissions_loading_django.py rename to scripts/07_benchmark_permissions_loading_django.py diff --git a/08_basic_email_web_crawler.py b/scripts/08_basic_email_web_crawler.py similarity index 100% rename from 08_basic_email_web_crawler.py rename to scripts/08_basic_email_web_crawler.py diff --git a/09_basic_link_web_crawler.py b/scripts/09_basic_link_web_crawler.py similarity index 100% rename from 09_basic_link_web_crawler.py rename to scripts/09_basic_link_web_crawler.py diff --git a/10_find_files_recursively.py b/scripts/10_find_files_recursively.py similarity index 92% rename from 10_find_files_recursively.py rename to scripts/10_find_files_recursively.py index 91cd73c..0c8e1eb 100755 --- a/10_find_files_recursively.py +++ b/scripts/10_find_files_recursively.py @@ -3,7 +3,7 @@ # constants PATH = './' -PATTERN = '*.py' +PATTERN = '*.md' def get_file_names(filepath, pattern): @@ -28,4 +28,4 @@ def output_files(list_of_files): if __name__ == '__main__': - all_files = get_file_names(PATH, PATTERN) + get_file_names(PATH, PATTERN) diff --git a/11_optimize_images_with_wand.py b/scripts/11_optimize_images_with_wand.py similarity index 100% rename from 11_optimize_images_with_wand.py rename to scripts/11_optimize_images_with_wand.py diff --git a/12_csv_split.py b/scripts/12_csv_split.py similarity index 100% rename from 12_csv_split.py rename to scripts/12_csv_split.py diff --git a/12_sample_csv.csv b/scripts/12_sample_csv.csv similarity index 100% rename from 12_sample_csv.csv rename to scripts/12_sample_csv.csv diff --git a/13_random_name_generator.py b/scripts/13_random_name_generator.py similarity index 100% rename from 13_random_name_generator.py rename to scripts/13_random_name_generator.py diff --git a/14_html_to_markdown.sh b/scripts/14_html_to_markdown.sh similarity index 100% rename from 14_html_to_markdown.sh rename to scripts/14_html_to_markdown.sh diff --git a/15_check_my_environment.py b/scripts/15_check_my_environment.py similarity index 100% rename from 15_check_my_environment.py rename to scripts/15_check_my_environment.py diff --git a/16_jinja_quick_load.py b/scripts/16_jinja_quick_load.py similarity index 100% rename from 16_jinja_quick_load.py rename to scripts/16_jinja_quick_load.py diff --git a/17_rewrite_git_history.md b/scripts/17_rewrite_git_history.md similarity index 100% rename from 17_rewrite_git_history.md rename to scripts/17_rewrite_git_history.md diff --git a/18_zipper.py b/scripts/18_zipper.py similarity index 100% rename from 18_zipper.py rename to scripts/18_zipper.py diff --git a/19_tsv-to-csv.py b/scripts/19_tsv-to-csv.py similarity index 100% rename from 19_tsv-to-csv.py rename to scripts/19_tsv-to-csv.py diff --git a/20_restore_file_from_git.py b/scripts/20_restore_file_from_git.py similarity index 100% rename from 20_restore_file_from_git.py rename to scripts/20_restore_file_from_git.py diff --git a/21_twitter_bot.py b/scripts/21_twitter_bot.py similarity index 100% rename from 21_twitter_bot.py rename to scripts/21_twitter_bot.py diff --git a/22_git_tag.py b/scripts/22_git_tag.py similarity index 100% rename from 22_git_tag.py rename to scripts/22_git_tag.py diff --git a/23_flask_session_test.py b/scripts/23_flask_session_test.py similarity index 100% rename from 23_flask_session_test.py rename to scripts/23_flask_session_test.py diff --git a/24_sql2csv.py b/scripts/24_sql2csv.py similarity index 100% rename from 24_sql2csv.py rename to scripts/24_sql2csv.py diff --git a/25_ip2geolocation.py b/scripts/25_ip2geolocation.py similarity index 100% rename from 25_ip2geolocation.py rename to scripts/25_ip2geolocation.py diff --git a/25_sample_csv.csv b/scripts/25_sample_csv.csv similarity index 100% rename from 25_sample_csv.csv rename to scripts/25_sample_csv.csv diff --git a/26_stock_scraper.py b/scripts/26_stock_scraper.py similarity index 100% rename from 26_stock_scraper.py rename to scripts/26_stock_scraper.py diff --git a/27_send_sms.py b/scripts/27_send_sms.py similarity index 100% rename from 27_send_sms.py rename to scripts/27_send_sms.py diff --git a/28_income_tax_calculator.py b/scripts/28_income_tax_calculator.py similarity index 100% rename from 28_income_tax_calculator.py rename to scripts/28_income_tax_calculator.py diff --git a/29_json_test.json b/scripts/29_json_test.json similarity index 100% rename from 29_json_test.json rename to scripts/29_json_test.json diff --git a/29_json_to_yaml.py b/scripts/29_json_to_yaml.py similarity index 100% rename from 29_json_to_yaml.py rename to scripts/29_json_to_yaml.py diff --git a/30_fullcontact.py b/scripts/30_fullcontact.py similarity index 100% rename from 30_fullcontact.py rename to scripts/30_fullcontact.py diff --git a/31_youtube_sentiment.py b/scripts/31_youtube_sentiment.py similarity index 100% rename from 31_youtube_sentiment.py rename to scripts/31_youtube_sentiment.py diff --git a/32_stock_scraper.py b/scripts/32_stock_scraper.py similarity index 100% rename from 32_stock_scraper.py rename to scripts/32_stock_scraper.py diff --git a/33_country_code.py b/scripts/33_country_code.py similarity index 100% rename from 33_country_code.py rename to scripts/33_country_code.py diff --git a/33_country_codes.json b/scripts/33_country_codes.json similarity index 100% rename from 33_country_codes.json rename to scripts/33_country_codes.json diff --git a/33_sample_csv.csv b/scripts/33_sample_csv.csv similarity index 100% rename from 33_sample_csv.csv rename to scripts/33_sample_csv.csv diff --git a/data.csv b/scripts/data.csv similarity index 100% rename from data.csv rename to scripts/data.csv From 14147e3dd0de7b7262e4617a37faaf0e05825665 Mon Sep 17 00:00:00 2001 From: Michael Herman Date: Sat, 17 Dec 2016 09:25:32 -0700 Subject: [PATCH 2/9] updated requirements.txt and added a todo --- TODO.md | 4 ++++ requirements.txt | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..598d675 --- /dev/null +++ b/TODO.md @@ -0,0 +1,4 @@ +1. Write unit and integration tests for *all* scripts +1. Add Travis +1. Add support for Python 2.7, 3.5, and 3.6 +1. Organize docs and folder structure better diff --git a/requirements.txt b/requirements.txt index b65ee68..2ad222b 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ beautifulsoup4==4.4.1 PyYAML==3.11 -requests==2.7.0 +requests==2.12.4 wheel==0.24.0 From 52cea88b4f524eddfee52ccb98db0e5bbd47d455 Mon Sep 17 00:00:00 2001 From: Michael Herman Date: Sat, 17 Dec 2016 09:28:23 -0700 Subject: [PATCH 3/9] added new script, updated readme, updated todo: --- TODO.md | 2 ++ readme.md | 1 + scripts/34_git_all_repos.py | 42 +++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 scripts/34_git_all_repos.py diff --git a/TODO.md b/TODO.md index 598d675..9358b94 100644 --- a/TODO.md +++ b/TODO.md @@ -2,3 +2,5 @@ 1. Add Travis 1. Add support for Python 2.7, 3.5, and 3.6 1. Organize docs and folder structure better +1. Add all scripts to single CLI for easy running, testing, and searching +1. Add License diff --git a/readme.md b/readme.md index 926be9d..59d5c64 100755 --- a/readme.md +++ b/readme.md @@ -32,3 +32,4 @@ 1. **30_fullcontact.py**: Call the [FullcContact](https://www.fullcontact.com/developer/) API 1. **31_youtube_sentiment.py**: Calculate sentiment score from the comments of a Youtube video 1. **32_stock_scraper.py**: Get stock prices +1. **34_git_all_repos.py**: Clone all repositories from a public user or organization on Github. Usage: `python git_all_repos.py users USER_NAME` or `python git_all_repos.py orgs ORG_NAME` diff --git a/scripts/34_git_all_repos.py b/scripts/34_git_all_repos.py new file mode 100644 index 0000000..b3e2d5b --- /dev/null +++ b/scripts/34_git_all_repos.py @@ -0,0 +1,42 @@ +import sys +import os +import requests + + +def get_total_repos(group, name): + repo_urls = [] + page = 1 + while True: + url = 'https://api.github.com/%7B0%7D/%7B1%7D/repos?per_page=100&page={2}' + r = requests.get(url.format(group, name, page)) + if r.status_code == 200: + rdata = r.json() + for repo in rdata: + repo_urls.append(repo['clone_url']) + if (len(rdata) >= 100): + page += 1 + else: + print('Found {0} repos.'.format(len(repo_urls))) + break + else: + print(r) + return False + return repo_urls + + +def clone_repos(all_repos): + count = 1 + print('Cloning...') + for repo in all_repos: + os.system('Git clone ' + repo) + print('Completed repo #{0} of {1}'.format(count, len(all_repos))) + count += 1 + +if __name__ == '__main__': + if len(sys.argv) > 2: + total = get_total_repos(sys.argv[1], sys.argv[2]) + if total: + clone_repos(total) + + else: + print('Usage: python USERS_OR_ORG GITHUB_USER_OR_ORG-NAME') From 2711f0ba1d08e28640653a5bfea1d0ca2b076ecf Mon Sep 17 00:00:00 2001 From: Skvm Date: Wed, 16 Aug 2017 19:19:48 +0100 Subject: [PATCH 4/9] Update requirements.txt 26_stock_scraper requires lxml --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2ad222b..beee86c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ beautifulsoup4==4.4.1 PyYAML==3.11 requests==2.12.4 wheel==0.24.0 +lxml==3.8.0 From 49153f1cd98ca9bd0102db41c010d76780fa7222 Mon Sep 17 00:00:00 2001 From: chetanya-shrimali Date: Sat, 16 Sep 2017 21:05:46 +0530 Subject: [PATCH 5/9] added license --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6f2e061 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Chetanya-Shrimali + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From b060e39b6111c41e3d5cef906ffb138726875f00 Mon Sep 17 00:00:00 2001 From: Chetanya-Shrimali Date: Sat, 16 Sep 2017 21:09:03 +0530 Subject: [PATCH 6/9] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 6f2e061..2ae0d76 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Chetanya-Shrimali +Copyright (c) 2017 realpython Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 8eb073544ac2d5866c2f86528c03b11409eec1cd Mon Sep 17 00:00:00 2001 From: Michael Herman Date: Sat, 16 Sep 2017 09:47:51 -0600 Subject: [PATCH 7/9] Update TODO.md --- TODO.md | 1 - 1 file changed, 1 deletion(-) diff --git a/TODO.md b/TODO.md index 9358b94..9d2dfea 100644 --- a/TODO.md +++ b/TODO.md @@ -3,4 +3,3 @@ 1. Add support for Python 2.7, 3.5, and 3.6 1. Organize docs and folder structure better 1. Add all scripts to single CLI for easy running, testing, and searching -1. Add License From 5b745282a923bee43b1bd3ee972a2351637b45ae Mon Sep 17 00:00:00 2001 From: uttamo Date: Thu, 28 Sep 2017 01:02:29 +0100 Subject: [PATCH 8/9] Use random.choice instead of random.randint random.choice is better for selecting random elements from a list than using random.randint to generate random indices to select --- scripts/13_random_name_generator.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/scripts/13_random_name_generator.py b/scripts/13_random_name_generator.py index 6f0a00a..0acb9bf 100755 --- a/scripts/13_random_name_generator.py +++ b/scripts/13_random_name_generator.py @@ -1,4 +1,4 @@ -from random import randint +from random import choice def random_name_generator(first, second, x): @@ -10,13 +10,8 @@ def random_name_generator(first, second, x): - number of random names """ names = [] - for i in range(0, int(x)): - random_first = randint(0, len(first)-1) - random_last = randint(0, len(second)-1) - names.append("{0} {1}".format( - first[random_first], - second[random_last]) - ) + for i in range(x): + names.append("{0} {1}".format(choice(first), choice(second))) return set(names) From cb448c2dc3593dbfbe1ca47b49193b320115aae5 Mon Sep 17 00:00:00 2001 From: Michael Herman Date: Sun, 25 Mar 2018 08:56:49 -0600 Subject: [PATCH 9/9] updated readme --- LICENSE | 2 +- readme.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 2ae0d76..3a6fa11 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 realpython +Copyright (c) 2018 Real Python Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index 59d5c64..78f4a72 100755 --- a/readme.md +++ b/readme.md @@ -32,4 +32,5 @@ 1. **30_fullcontact.py**: Call the [FullcContact](https://www.fullcontact.com/developer/) API 1. **31_youtube_sentiment.py**: Calculate sentiment score from the comments of a Youtube video 1. **32_stock_scraper.py**: Get stock prices +1. **33_country_code.py**: Convert country code to country name 1. **34_git_all_repos.py**: Clone all repositories from a public user or organization on Github. Usage: `python git_all_repos.py users USER_NAME` or `python git_all_repos.py orgs ORG_NAME`