Delete VulnerabilityLog and print_report, make formatters/text.py and json.py, -j for JSON option, refactor CALL_IDENTIFIER to tilde because of /uUGLYNUMBERS json output, refactor super calls of the subclasses of Vulnerability to be silky smooth with kwargs, implement as_dict methods for Node and vulnerability types, make tests pass with these changes, fixed a few flake8y things

Author: KevinHock

func_counter.py

@@ -20,14 +20,14 @@ def visit_Call(self, node):
 
     def visit_FunctionDef(self, node):
         if node.name in functions:
-            node.name += '¤'
+            node.name += '~'
         functions[node.name] = len(node.body)
         for n in node.body:
             self.visit(n)
 
     def visit_ClassDef(self, node):
         if node.name in classes:
-            node.name += '¤'
+            node.name += '~'
         classes[node.name] = len(node.body)
         for n in node.body:
             self.visit(n)

pyt/__main__.py

@@ -21,6 +21,10 @@
 from .draw import draw_cfgs, draw_lattices
 from .expr_visitor import make_cfg
 from .fixed_point import analyse
+from .formatters import (
+    json,
+    text
+)
 from .framework_adaptor import FrameworkAdaptor
 from .framework_helper import (
     is_django_view_function,
@@ -93,7 +97,7 @@ def parse_args(args):
                         help='Input trigger word file.',
                         type=str,
                         default=default_trigger_word_file)
-    parser.add_argument('-b', '--blackbox-mapping-file',
+    parser.add_argument('-m', '--blackbox-mapping-file',
                         help='Input blackbox mapping file.',
                         type=str,
                         default=default_blackbox_mapping_file)
@@ -111,6 +115,10 @@ def parse_args(args):
                         ' create a database.', action='store_true')
     parser.add_argument('-dl', '--draw-lattice',
                         nargs='+', help='Draws a lattice.')
+    parser.add_argument('-j', '--json',
+                        help='Prints JSON instead of report.',
+                        action='store_true',
+                        default=False)
 
     analysis_group = parser.add_mutually_exclusive_group()
     analysis_group.add_argument('-li', '--liveness',
@@ -177,7 +185,7 @@ def parse_args(args):
     return parser.parse_args(args)
 
 
-def analyse_repo(github_repo, analysis_type, ui_mode):
+def analyse_repo(args, github_repo, analysis_type, ui_mode):
     cfg_list = list()
     directory = os.path.dirname(github_repo.path)
     project_modules = get_modules(directory)
@@ -193,7 +201,7 @@ def analyse_repo(github_repo, analysis_type, ui_mode):
 
     initialize_constraint_table(cfg_list)
     analyse(cfg_list, analysis_type=analysis_type)
-    vulnerability_log = find_vulnerabilities(
+    vulnerabilities = find_vulnerabilities(
         cfg_list,
         analysis_type,
         ui_mode,
@@ -202,7 +210,7 @@ def analyse_repo(github_repo, analysis_type, ui_mode):
             args.trigger_word_file
         )
     )
-    return vulnerability_log
+    return vulnerabilities
 
 
 def main(command_line_args=sys.argv[1:]):
@@ -225,9 +233,12 @@ def main(command_line_args=sys.argv[1:]):
         repos = get_repos(args.git_repos)
         for repo in repos:
             repo.clone()
-            vulnerability_log = analyse_repo(repo, analysis, ui_mode)
-            vulnerability_log.print_report()
-            if not vulnerability_log.vulnerabilities:
+            vulnerabilities = analyse_repo(args, repo, analysis, ui_mode)
+            if args.json:
+                json.report(vulnerabilities, sys.stdout)
+            else:
+                text.report(vulnerabilities, sys.stdout)
+            if not vulnerabilities:
                 repo.clean_up()
         exit()
 
@@ -239,7 +250,8 @@ def main(command_line_args=sys.argv[1:]):
             analysis,
             analyse_repo,
             args.csv_path,
-            ui_mode
+            ui_mode,
+            args
         )
         exit()
 
@@ -278,7 +290,7 @@ def main(command_line_args=sys.argv[1:]):
 
     analyse(cfg_list, analysis_type=analysis)
 
-    vulnerability_log = find_vulnerabilities(
+    vulnerabilities = find_vulnerabilities(
         cfg_list,
         analysis,
         ui_mode,
@@ -287,17 +299,20 @@ def main(command_line_args=sys.argv[1:]):
             args.trigger_word_file
         )
     )
-    vulnerability_log.print_report()
+    if args.json:
+        json.report(vulnerabilities, sys.stdout)
+    else:
+        text.report(vulnerabilities, sys.stdout)
 
     if args.draw_cfg:
         if args.output_filename:
             draw_cfgs(cfg_list, args.output_filename)
         else:
             draw_cfgs(cfg_list)
     if args.print:
-        l = print_lattice(cfg_list, analysis)
+        lattice = print_lattice(cfg_list, analysis)
 
-        print_table(l)
+        print_table(lattice)
         for i, e in enumerate(cfg_list):
             print('############## CFG number: ', i)
             print(e)
@@ -311,7 +326,7 @@ def main(command_line_args=sys.argv[1:]):
         pprint(project_modules)
 
     if args.create_database:
-        create_database(cfg_list, vulnerability_log)
+        create_database(cfg_list, vulnerabilities)
     if args.draw_lattice:
         draw_lattices(cfg_list)
 
@@ -325,7 +340,7 @@ def main(command_line_args=sys.argv[1:]):
             cfg_to_file(cfg_list)
             verbose_cfg_to_file(cfg_list)
             lattice_to_file(cfg_list, analysis)
-            vulnerabilities_to_file(vulnerability_log)
+            vulnerabilities_to_file(vulnerabilities)
         else:
             if args.def_use_chain:
                 def_use_chain_to_file(cfg_list)
@@ -338,7 +353,7 @@ def main(command_line_args=sys.argv[1:]):
             if args.lattice:
                 lattice_to_file(cfg_list, analysis)
             if args.vulnerabilities:
-                vulnerabilities_to_file(vulnerability_log)
+                vulnerabilities_to_file(vulnerabilities)
 
 
 if __name__ == '__main__':

pyt/analysis_base.py

@@ -1,5 +1,9 @@
 """This module contains a base class for the analysis component used in PyT."""
-from abc import ABCMeta, abstractmethod
+
+from abc import (
+    ABCMeta,
+    abstractmethod
+)
 
 
 class AnalysisBase(metaclass=ABCMeta):

pyt/ast_helper.py

@@ -1,5 +1,6 @@
 """This module contains helper function.
 Useful when working with the ast module."""
+
 import ast
 import os
 import subprocess

pyt/draw.py

@@ -1,4 +1,5 @@
 """Draws CFG."""
+
 import argparse
 from itertools import permutations
 from subprocess import call
@@ -103,7 +104,6 @@ def draw_cfg(cfg, output_filename='output'):
 
 
 class Node():
-
     def __init__(self, s, parent, children=None):
         self.s = s
         self.parent = parent

pyt/expr_visitor.py

@@ -451,7 +451,7 @@ def return_handler(
         for node in function_nodes:
             # Only `Return`s and `Raise`s can be of type ConnectToExitNode
             if isinstance(node, ConnectToExitNode):
-                # Create e.g. ¤call_1 = ret_func_foo RestoreNode
+                # Create e.g. ~call_1 = ret_func_foo RestoreNode
                 LHS = CALL_IDENTIFIER + 'call_' + str(saved_function_call_index)
                 RHS = 'ret_' + get_call_names_as_string(call_node.func)
                 return_node = RestoreNode(
@@ -478,11 +478,11 @@ def process_function(self, call_node, definition):
         Visit and get function nodes. (visit_and_get_function_nodes)
         Loop through each save_N_LHS node and create an e.g.
             foo = save_1_foo or, if foo was a call arg, foo = arg_mapping[foo]. (restore_saved_local_scope)
-        Create e.g. ¤call_1 = ret_func_foo RestoreNode. (return_handler)
+        Create e.g. ~call_1 = ret_func_foo RestoreNode. (return_handler)
 
         Notes:
             Page 31 in the original thesis, but changed a little.
-            We don't have to return the ¤call_1 = ret_func_foo RestoreNode made in return_handler,
+            We don't have to return the ~call_1 = ret_func_foo RestoreNode made in return_handler,
                 because it's the last node anyway, that we return in this function.
             e.g. ret_func_foo gets assigned to visit_Return.
 

pyt/formatters/json.py

@@ -0,0 +1,33 @@
+"""This formatter outputs the issues in JSON."""
+
+import json
+from datetime import datetime
+
+
+def report(
+    vulnerabilities,
+    fileobj
+):
+    """
+    Prints issues in JSON format.
+
+    Args:
+        vulnerabilities: list of vulnerabilities to report
+        fileobj: The output file object, which may be sys.stdout
+    """
+    TZ_AGNOSTIC_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+    time_string = datetime.utcnow().strftime(TZ_AGNOSTIC_FORMAT)
+
+    machine_output = {
+        'generated_at': time_string,
+        'vulnerabilities': [vuln.as_dict() for vuln in vulnerabilities]
+    }
+
+    result = json.dumps(
+        machine_output,
+        indent=4,
+        sort_keys=True
+    )
+
+    with fileobj:
+        fileobj.write(result)

pyt/formatters/text.py

@@ -0,0 +1,23 @@
+"""This formatter outputs the issues as plain text."""
+
+
+def report(
+	vulnerabilities,
+	fileobj
+):
+	"""
+	Prints issues in text format.
+
+	Args:
+		vulnerabilities: list of vulnerabilities to report
+		fileobj: The output file object, which may be sys.stdout
+	"""
+	number_of_vulnerabilities = len(vulnerabilities)
+	with fileobj:
+		if number_of_vulnerabilities == 1:
+			fileobj.write('%s vulnerability found:' % number_of_vulnerabilities)
+		else:
+			fileobj.write('%s vulnerabilities found:' % number_of_vulnerabilities)
+
+		for i, vulnerability in enumerate(vulnerabilities, start=1):
+			fileobj.write('Vulnerability {}:\n{}\n'.format(i, vulnerability))

pyt/framework_adaptor.py

@@ -1,13 +1,11 @@
 """A generic framework adaptor that leaves route criteria to the caller."""
+
 import ast
 
 from .ast_helper import Arguments
 from .expr_visitor import make_cfg
 from .module_definitions import project_definitions
-from .node_types import (
-    AssignmentNode,
-    TaintedNode
-)
+from .node_types import TaintedNode
 
 
 class FrameworkAdaptor():

pyt/github_search.py

@@ -197,7 +197,7 @@ def get_dates(start_date, end_date=date.today(), interval=7):
                                       delta.days % interval))
 
 
-def scan_github(search_string, start_date, analysis_type, analyse_repo_func, csv_path, ui_mode):
+def scan_github(search_string, start_date, analysis_type, analyse_repo_func, csv_path, ui_mode, other_args):
     analyse_repo = analyse_repo_func
     for d in get_dates(start_date, interval=7):
         q = Query(SEARCH_REPO_URL, search_string,
@@ -214,27 +214,27 @@ def scan_github(search_string, start_date, analysis_type, analyse_repo_func, csv
                 try:
                     r.clone()
                 except NoEntryPathError as err:
-                    save_repo_scan(repo, r.path, vulnerability_log=None, error=err)
+                    save_repo_scan(repo, r.path, vulnerabilities=None, error=err)
                     continue
                 except:
-                    save_repo_scan(repo, r.path, vulnerability_log=None, error='Other Error Unknown while cloning :-(')
+                    save_repo_scan(repo, r.path, vulnerabilities=None, error='Other Error Unknown while cloning :-(')
                     continue
                 try:
-                    vulnerability_log = analyse_repo(r, analysis_type, ui_mode)
-                    if vulnerability_log.vulnerabilities:
-                        save_repo_scan(repo, r.path, vulnerability_log)
+                    vulnerabilities = analyse_repo(other_args, r, analysis_type, ui_mode)
+                    if vulnerabilities:
+                        save_repo_scan(repo, r.path, vulnerabilities)
                         add_repo_to_csv(csv_path, r)
                     else:
-                        save_repo_scan(repo, r.path, vulnerability_log=None)
+                        save_repo_scan(repo, r.path, vulnerabilities=None)
                     r.clean_up()
                 except SyntaxError as err:
-                    save_repo_scan(repo, r.path, vulnerability_log=None, error=err)
+                    save_repo_scan(repo, r.path, vulnerabilities=None, error=err)
                 except IOError as err:
-                    save_repo_scan(repo, r.path, vulnerability_log=None, error=err)
+                    save_repo_scan(repo, r.path, vulnerabilities=None, error=err)
                 except AttributeError as err:
-                    save_repo_scan(repo, r.path, vulnerability_log=None, error=err)
+                    save_repo_scan(repo, r.path, vulnerabilities=None, error=err)
                 except:
-                    save_repo_scan(repo, r.path, vulnerability_log=None, error='Other Error Unknown :-(')
+                    save_repo_scan(repo, r.path, vulnerabilities=None, error='Other Error Unknown :-(')
 
 if __name__ == '__main__':
     for x in get_dates(date(2010, 1, 1), interval=93):