From fedcc3effee0e61584a89f7d8213ef83468d0760 Mon Sep 17 00:00:00 2001 From: A D Date: Mon, 10 Jun 2019 01:51:47 -0700 Subject: [PATCH] Adding lesson10 homework --- .../lesson10/assignment/data/customers.csv | 11 + .../lesson10/assignment/data/products.csv | 11 + .../lesson10/assignment/data/rentals.csv | 10 + students/aaron/lesson10/assignment/pylint.log | 83 ++++++ students/aaron/lesson10/assignment/pylintrc | 236 ++++++++++++++++++ students/aaron/lesson10/assignment/pytest.log | 12 + .../aaron/lesson10/assignment/src/database.py | 140 +++++++++++ .../assignment/tests/test_database.py | 49 ++++ students/aaron/lesson10/assignment/timing.log | 36 +++ 9 files changed, 588 insertions(+) create mode 100755 students/aaron/lesson10/assignment/data/customers.csv create mode 100755 students/aaron/lesson10/assignment/data/products.csv create mode 100755 students/aaron/lesson10/assignment/data/rentals.csv create mode 100644 students/aaron/lesson10/assignment/pylint.log create mode 100644 students/aaron/lesson10/assignment/pylintrc create mode 100644 students/aaron/lesson10/assignment/pytest.log create mode 100755 students/aaron/lesson10/assignment/src/database.py create mode 100755 students/aaron/lesson10/assignment/tests/test_database.py create mode 100644 students/aaron/lesson10/assignment/timing.log diff --git a/students/aaron/lesson10/assignment/data/customers.csv b/students/aaron/lesson10/assignment/data/customers.csv new file mode 100755 index 0000000..a77e7d6 --- /dev/null +++ b/students/aaron/lesson10/assignment/data/customers.csv @@ -0,0 +1,11 @@ +user_id,name,address,zip_code,phone_number,email +user001,Elisa Miles,4490 Union Street,98109,206-922-0882,elisa.miles@yahoo.com +user002,Maya Data,4936 Elliot Avenue,98115,206-777-1927,mdata@uw.edu +user003,Andy Norris,348 Terra Street,98501,206-309-2533,andy.norris@gmail.com +user004,Flor Matatena,885 Boone Crockett Lane,97209,206-414-2629,matseattle@pge.com +user005,Dan Sounders,861 Honeysuckle Lane,98244,206-279-1723,soundersoccer@mls.com +user006,Leo Dembele,2725 Mutton Town Road,98368,206-203-1294,leo.dembele@comcast.com +user007,Pete Nicholas,668 Elliot Avenue,98115,206-279-8759,nicholasp@amazon.com +user008,Shirlene Harris,4329 Honeysuckle Lane,98055,206-279-5340,harrisfamily@gmail.com +user009,Nick Rather,4679 Goodwin Avenue,98619,206-777-1965,nick.rather@microsoft.com +user010,Jose Garza,2717 Raccoon Run,98116,206-946-8200,joegarza@boeing.com diff --git a/students/aaron/lesson10/assignment/data/products.csv b/students/aaron/lesson10/assignment/data/products.csv new file mode 100755 index 0000000..527ca98 --- /dev/null +++ b/students/aaron/lesson10/assignment/data/products.csv @@ -0,0 +1,11 @@ +product_id,description,product_type,quantity_available +prd001,60-inch TV stand,livingroom,3 +prd002,L-shaped sofa,livingroom,0 +prd003,Acacia kitchen table,kitchen,7 +prd004,Queen bed,bedroom,10 +prd005,Reading lamp,bedroom,20 +prd006,Portable heater,bathroom,14 +prd007,Ballerina painting,livingroom,0 +prd008,Smart microwave,kitchen,30 +prd009,Popcorn machine,kitchen,0 +prd010,60-inch TV,livingroom,3 diff --git a/students/aaron/lesson10/assignment/data/rentals.csv b/students/aaron/lesson10/assignment/data/rentals.csv new file mode 100755 index 0000000..668c845 --- /dev/null +++ b/students/aaron/lesson10/assignment/data/rentals.csv @@ -0,0 +1,10 @@ +product_id,user_id +prd003,user004 +prd002,user008 +prd002,user005 +prd005,user001 +prd010,user002 +prd007,user002 +prd006,user003 +prd005,user003 +prd001,user010 diff --git a/students/aaron/lesson10/assignment/pylint.log b/students/aaron/lesson10/assignment/pylint.log new file mode 100644 index 0000000..ec6ee91 --- /dev/null +++ b/students/aaron/lesson10/assignment/pylint.log @@ -0,0 +1,83 @@ +# python3 -m pylint --rcfile=pylintrc src/ >pylint.log 2>&1 + + +Report +====== +101 statements analysed. + +Statistics by type +------------------ + ++---------+-------+-----------+-----------+------------+---------+ +|type |number |old number |difference |%documented |%badname | ++=========+=======+===========+===========+============+=========+ +|module |1 |1 |= |100.00 |0.00 | ++---------+-------+-----------+-----------+------------+---------+ +|class |0 |0 |= |0 |0 | ++---------+-------+-----------+-----------+------------+---------+ +|method |0 |0 |= |0 |0 | ++---------+-------+-----------+-----------+------------+---------+ +|function |14 |14 |= |100.00 |0.00 | ++---------+-------+-----------+-----------+------------+---------+ + + + +External dependencies +--------------------- +:: + + pymongo (database) + + + +Raw metrics +----------- + ++----------+-------+------+---------+-----------+ +|type |number |% |previous |difference | ++==========+=======+======+=========+===========+ +|code |109 |76.76 |109 |= | ++----------+-------+------+---------+-----------+ +|docstring |15 |10.56 |15 |= | ++----------+-------+------+---------+-----------+ +|comment |3 |2.11 |3 |= | ++----------+-------+------+---------+-----------+ +|empty |15 |10.56 |15 |= | ++----------+-------+------+---------+-----------+ + + + +Duplication +----------- + ++-------------------------+------+---------+-----------+ +| |now |previous |difference | ++=========================+======+=========+===========+ +|nb duplicated lines |0 |0 |= | ++-------------------------+------+---------+-----------+ +|percent duplicated lines |0.000 |0.000 |= | ++-------------------------+------+---------+-----------+ + + + +Messages by category +-------------------- + ++-----------+-------+---------+-----------+ +|type |number |previous |difference | ++===========+=======+=========+===========+ +|convention |0 |0 |= | ++-----------+-------+---------+-----------+ +|refactor |0 |0 |= | ++-----------+-------+---------+-----------+ +|warning |0 |0 |= | ++-----------+-------+---------+-----------+ +|error |0 |0 |= | ++-----------+-------+---------+-----------+ + + + + +-------------------------------------------------------------------- +Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) + diff --git a/students/aaron/lesson10/assignment/pylintrc b/students/aaron/lesson10/assignment/pylintrc new file mode 100644 index 0000000..c8e5fee --- /dev/null +++ b/students/aaron/lesson10/assignment/pylintrc @@ -0,0 +1,236 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add to the black list. It should be a base name, not a +# path. You may set this option multiple times. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifier separated by comma (,) or put this option +# multiple time. +disable= too-few-public-methods, too-many-arguments + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Include message's id in output +include-ids=no + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (R0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (R0004). +comment=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match functions or classes name which do +# not require a docstring +no-docstring-rgx=__.*__ + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. +generated-members=REQUEST,acl_users,aq_parent + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branchs=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,string,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp diff --git a/students/aaron/lesson10/assignment/pytest.log b/students/aaron/lesson10/assignment/pytest.log new file mode 100644 index 0000000..c18ee7d --- /dev/null +++ b/students/aaron/lesson10/assignment/pytest.log @@ -0,0 +1,12 @@ +# python3 -m pytest -vvv tests/ >pytest.log 2>&1 +============================= test session starts ============================== +platform darwin -- Python 3.7.2, pytest-4.3.0, py-1.7.0, pluggy-0.8.1 -- /usr/local/opt/python/bin/python3.7 +cachedir: .pytest_cache +rootdir: /Users/adevey/class/Python220A_2019/students/aaron/lesson10/assignment, inifile: +collecting ... collected 3 items + +tests/test_database.py::test_import_data PASSED [ 33%] +tests/test_database.py::test_show_available_products PASSED [ 66%] +tests/test_database.py::test_show_rentals PASSED [100%] + +=========================== 3 passed in 0.37 seconds =========================== diff --git a/students/aaron/lesson10/assignment/src/database.py b/students/aaron/lesson10/assignment/src/database.py new file mode 100755 index 0000000..23b1c55 --- /dev/null +++ b/students/aaron/lesson10/assignment/src/database.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +''' Stores and retrieves data from a rentals db ''' + +import os +from timeit import default_timer as timer +import pymongo + +CLIENT = pymongo.MongoClient('localhost', 27017) +DB = CLIENT.data +LOG_FILE = 'timing.log' + +def log_timing(func_name, time, rows=0): + ''' A quick function to log timings to a file ''' + with open(LOG_FILE, 'a+') as fhand: + fhand.write("Function=%s timing=%s rows=%s\n" % (func_name, time, rows)) + +def timing_logger_standard(row_hint): + ''' Decorator for standard functions with fixed or no rows ''' + def wrap(func): + ''' The inner decorator in a decorator with args ''' + def add_timing(*args): + ''' The actual decorating function ''' + start = timer() + (data) = func(*args) + end = timer() + timing = end - start + log_timing(func.__name__, timing, row_hint) + return data + return add_timing + return wrap + +def timing_logger_find(func): + ''' Decorator for functions which find rows ''' + def add_timing(*args): + ''' The decorating function ''' + start = timer() + data = func(*args) + end = timer() + timing = end - start + log_timing(func.__name__, timing, len(data)) + return data + return add_timing + +def timing_logger_import(func): + ''' Decorator for functions which import rows from csv ''' + def add_timing(*args): + ''' The decorating function ''' + start = timer() + (records, errors) = func(*args) + end = timer() + timing = end - start + log_timing(func.__name__, timing, records + errors) + return (records, errors) + return add_timing + +@timing_logger_standard('N/A') +def setup_db(database): + ''' sets up the collections and returns them ''' + products = database.products + products.drop() + products.create_index([('product_id', pymongo.ASCENDING)], unique=True) + customers = database.customers + customers.drop() + customers.create_index([('user_id', pymongo.ASCENDING)], unique=True) + rentals = database.rentals + rentals.drop() + return (products, customers, rentals) + +@timing_logger_standard('N/A') +def import_data(dirname, products, customers, rentals): + ''' Imports data from a bunch of csv files located at dirname ''' + prod_rec_errs = import_csv_data(PRODUCTS, os.path.join(dirname, products)) + cust_rec_errs = import_csv_data(CUSTOMERS, os.path.join(dirname, customers)) + rent_rec_errs = import_csv_data(RENTALS, os.path.join(dirname, rentals)) + zipped = list(zip(prod_rec_errs, cust_rec_errs, rent_rec_errs)) + return zipped + +@timing_logger_standard(1) +def insert_row(collection, row): + ''' Inserts a row into DB. Returns True on success, else False ''' + try: + collection.insert_one(row) + except pymongo.errors.PyMongoError as err: + print("Failure to write row to db. %s - %s", row, err) + return False + return True + +@timing_logger_import +def import_csv_data(collection, path): + ''' Imports csv data from a file, returns (records, failures) ''' + columns = [] + records = 0 + errors = 0 + try: + with open(path, encoding="ISO-8859-1") as fhand: + # retrieve the csv header + header = fhand.readline().rstrip() + columns = header.split(',') + + # retrieve each row and populate data + rows = [] + for line in fhand: + fields = line.rstrip().split(',') + row = {} + for field in enumerate(fields): + row[columns[field[0]]] = field[1] + rows.append(row) + except Exception as err: + print('Failure to read data from %s: %s', path, err) + raise err + for row in rows: + success = insert_row(collection, row) + if success: + records += 1 + else: + errors += 1 + return (records, errors) + +@timing_logger_find +def show_available_products(): + ''' returns all available products (quantity isn't 0) ''' + prods = {} + for product in PRODUCTS.find({'quantity_available': {'$ne': "0"}}): + product_id = product.pop('product_id') + product.pop('_id') + prods[product_id] = product + return prods + +@timing_logger_find +def show_rentals(product_id): + ''' returns all of the customers who rented a product ''' + rents = {} + for rent in RENTALS.find({'product_id': product_id}): + for customer in CUSTOMERS.find({'user_id': rent['user_id']}): + cust_id = customer.pop('user_id') + customer.pop('_id') + rents[cust_id] = customer + return rents + +PRODUCTS, CUSTOMERS, RENTALS = setup_db(DB) diff --git a/students/aaron/lesson10/assignment/tests/test_database.py b/students/aaron/lesson10/assignment/tests/test_database.py new file mode 100755 index 0000000..a7420a0 --- /dev/null +++ b/students/aaron/lesson10/assignment/tests/test_database.py @@ -0,0 +1,49 @@ +""" +grade lesson 5 +""" + +import os +import pytest + +import src.database as l + +@pytest.fixture +def _show_available_products(): + return { + 'prd001': {'description': '60-inch TV stand', 'product_type': 'livingroom', 'quantity_available': '3'}, + 'prd003': {'description': 'Acacia kitchen table', 'product_type': 'kitchen', 'quantity_available': '7'}, + 'prd004': {'description': 'Queen bed', 'product_type': 'bedroom', 'quantity_available': '10'}, + 'prd005': {'description': 'Reading lamp', 'product_type': 'bedroom', 'quantity_available': '20'}, + 'prd006': {'description': 'Portable heater', 'product_type': 'bathroom', 'quantity_available': '14'}, + 'prd008': {'description': 'Smart microwave', 'product_type': 'kitchen', 'quantity_available': '30'}, + 'prd010': {'description': '60-inch TV', 'product_type': 'livingroom', 'quantity_available': '3'}} + +@pytest.fixture +def _show_rentals(): + return { + 'user001': {'name': 'Elisa Miles', 'address': '4490 Union Street', 'zip_code': '98109', 'phone_number': '206-922-0882', 'email': 'elisa.miles@yahoo.com'}, + 'user003': {'name': 'Andy Norris', 'address': '348 Terra Street', 'zip_code': '98501', 'phone_number': '206-309-2533', 'email': 'andy.norris@gmail.com'}} + +def test_import_data(): + """ import """ + data_dir = 'data' + added, errors = l.import_data(data_dir, "products.csv", "customers.csv", "rentals.csv") + + for add in added: + assert isinstance(add, int) + + for error in errors: + assert isinstance(error, int) + + assert added == (10, 10, 9) + assert errors == (0, 0, 0) + +def test_show_available_products(_show_available_products): + """ available products """ + students_response = l.show_available_products() + assert students_response == _show_available_products + +def test_show_rentals(_show_rentals): + """ rentals """ + students_response = l.show_rentals("prd005") + assert students_response == _show_rentals diff --git a/students/aaron/lesson10/assignment/timing.log b/students/aaron/lesson10/assignment/timing.log new file mode 100644 index 0000000..db570ee --- /dev/null +++ b/students/aaron/lesson10/assignment/timing.log @@ -0,0 +1,36 @@ +Function=setup_db timing=0.24530950199999996 rows=N/A +Function=insert_row timing=0.0009468180000000714 rows=1 +Function=insert_row timing=0.0004227769999999298 rows=1 +Function=insert_row timing=0.00039583700000001 rows=1 +Function=insert_row timing=0.000467960000000045 rows=1 +Function=insert_row timing=0.00038591100000007206 rows=1 +Function=insert_row timing=0.00039101400000007835 rows=1 +Function=insert_row timing=0.0003918800000000111 rows=1 +Function=insert_row timing=0.0003754830000000098 rows=1 +Function=insert_row timing=0.0003770619999999836 rows=1 +Function=insert_row timing=0.0003823560000000281 rows=1 +Function=import_csv_data timing=0.006411527000000028 rows=10 +Function=insert_row timing=0.0008050619999999675 rows=1 +Function=insert_row timing=0.00038199199999999767 rows=1 +Function=insert_row timing=0.00034650000000002734 rows=1 +Function=insert_row timing=0.0003463940000000276 rows=1 +Function=insert_row timing=0.00034458999999997797 rows=1 +Function=insert_row timing=0.00032898400000003214 rows=1 +Function=insert_row timing=0.00044731799999997435 rows=1 +Function=insert_row timing=0.00039740500000007284 rows=1 +Function=insert_row timing=0.00036246700000008847 rows=1 +Function=insert_row timing=0.0003686210000000134 rows=1 +Function=import_csv_data timing=0.005952216999999926 rows=10 +Function=insert_row timing=0.045516927000000096 rows=1 +Function=insert_row timing=0.0007956620000000303 rows=1 +Function=insert_row timing=0.0006306070000000608 rows=1 +Function=insert_row timing=0.0005461449999999424 rows=1 +Function=insert_row timing=0.0005180769999999502 rows=1 +Function=insert_row timing=0.0005042979999999586 rows=1 +Function=insert_row timing=0.0004834340000000603 rows=1 +Function=insert_row timing=0.0005127179999999676 rows=1 +Function=insert_row timing=0.00039621499999997756 rows=1 +Function=import_csv_data timing=0.05208484099999999 rows=9 +Function=import_data timing=0.0649006150000001 rows=N/A +Function=show_available_products timing=0.0006120530000000013 rows=7 +Function=show_rentals timing=0.0013412259999999732 rows=2