AML-6 Model Manifest Compilation Integration (#113)

* first commit (CLI function)

* wip

* initial functional commit with Compilation

* added CLI endpoint to CLI tool

* moved subparser before parser arg processing

* renamed CLI function compile to lock, fixed grammar issue

* renamed lock to freeze, addressed bug detected by lemonez

* removed unnecessary import

Author: zeryx

Algorithmia/CLI.py

@@ -2,10 +2,11 @@
 import os
 from Algorithmia.errors import DataApiError
 from Algorithmia.algo_response import AlgoResponse
+from Algorithmia.util import md5_for_file, md5_for_str
 import json, re, requests, six
 import toml
 import shutil
-
+from time import time
 
 class CLI:
     def __init__(self):
@@ -101,7 +102,7 @@ def runalgo(self, options, client):
         elif (options.binary_file != None):
             # binary file
             with open(options.binary_file, "rb") as f:
-                algo_inputs = bytes(f.read())
+                algo_input = bytes(f.read())
             key = self.getAPIkey(options.profile)
             content = 'application/octet-stream'
 
@@ -245,6 +246,31 @@ def cat(self, path, client):
 
         return result
 
+    # algo freeze
+    def freezeAlgo(self, client, manifest_path="model_manifest.json"):
+        if os.path.exists(manifest_path):
+            with open(manifest_path, 'r') as f:
+                manifest_file = json.load(f)
+            manifest_file['timestamp'] = str(time())
+            required_files = manifest_file['required_files']
+            optional_files = manifest_file['optional_files']
+            for i in range(len(required_files)):
+                uri = required_files[i]['source_uri']
+                local_file = client.file(uri).getFile(as_path=True)
+                md5_checksum = md5_for_file(local_file)
+                required_files[i]['md5_checksum'] = md5_checksum
+            for i in range(len(optional_files)):
+                uri = required_files[i]['source_uri']
+                local_file = client.file(uri).getFile(as_path=True)
+                md5_checksum = md5_for_file(local_file)
+                required_files[i]['md5_checksum'] = md5_checksum
+            lock_md5_checksum = md5_for_str(str(manifest_file))
+            manifest_file['lock_checksum'] = lock_md5_checksum
+            with open('model_manifest.json.freeze', 'w') as f:
+                json.dump(manifest_file, f)
+        else:
+            print("Expected to find a model_manifest.json file, none was discovered in working directory")
+
     # algo cp <src> <dest>
     def cp(self, src, dest, client):
 

Algorithmia/__main__.py

@@ -108,7 +108,7 @@ def main():
     parser_cat.add_argument('--profile', action = 'store', type = str, default = 'default')
 
     #sub parser for getting environment template
-    parser_template = subparsers.add_parser('template',help='template <envid> <dest> downloads an environment template to the destination')
+    parser_template = subparsers.add_parser('template', help='template <envid> <dest> downloads an environment template to the destination')
     parser_template.add_argument('envid',help='environment specification id')
     parser_template.add_argument('dest',help='destination for template download')
 
@@ -130,8 +130,12 @@ def main():
     subparsers.add_parser('help')
     parser.add_argument('--profile', action = 'store', type = str, default = 'default')
 
+    #sub parser for freeze
+    subparsers.add_parser('freeze', help="freezes a model_manifest.json file into a model_manifest.json.freeze")
+    
     args = parser.parse_args()
 
+
     #run auth before trying to create a client
     if args.cmd == 'auth':
 
@@ -215,6 +219,9 @@ def main():
     elif args.cmd == 'builds':
         print(CLI().getBuildLogs(args.user, args.algo, client))
 
+    elif args.cmd == "freeze":
+        print(CLI().freezeAlgo(client))
+
     else:
         parser.parse_args(['-h'])
 

Algorithmia/util.py

@@ -1,8 +1,10 @@
 import re
-from Algorithmia.errors import DataApiError
+import hashlib
 
 FNAME_MATCH = re.compile(r'/([^/]+)$')  # From the last slash to the end of the string 
-PREFIX = re.compile(r'([^:]+://)(/)?(.+)')   # Check for a prefix like data://
+PREFIX = re.compile(r'([^:]+://)(/)?(.+)')  # Check for a prefix like data://
+
+
 def getParentAndBase(path):
     match = PREFIX.match(path)
     if match is None:
@@ -26,7 +28,22 @@ def getParentAndBase(path):
             parent_path = '{prefix}{uri}'.format(prefix=prefix, uri='/'.join(parts[:-1]))
         return parent_path, parts[-1]
 
+
 def pathJoin(parent, base):
     if parent.endswith('/'):
         return parent + base
     return parent + '/' + base
+
+
+def md5_for_file(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return str(hash_md5.hexdigest())
+
+
+def md5_for_str(content):
+    hash_md5 = hashlib.md5()
+    hash_md5.update(content.encode())
+    return str(hash_md5.hexdigest())