From 215fb6414340e9a17e477bf448874f749a32f3b0 Mon Sep 17 00:00:00 2001 From: Travis McPeak Date: Tue, 1 Sep 2015 10:22:09 -0700 Subject: [PATCH] Adding test tool for check OpenStack projects' Bandit job This commit adds to the openstack coverage tool. Specifically we add a '-t' option. If provided, this option will git clone any project which uses Bandit in a job, run the tox Bandit job, capture the output of any failed run, and display a summary table at the end. This tool is to support pre-release Bandit checking to make sure that our changes haven't introduced any new issues in projects which use it. Change-Id: I321bcb15b59e3ee00ed2f2c6c2c890b77f30370e --- tools/openstack_coverage.py | 161 +++++++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 4 deletions(-) diff --git a/tools/openstack_coverage.py b/tools/openstack_coverage.py index dd5cb338..972ce755 100755 --- a/tools/openstack_coverage.py +++ b/tools/openstack_coverage.py @@ -21,19 +21,28 @@ within the openstack-infra/project-config repository. Parses out Bandit jobs and tests as defined within these configurations. Prints the summary of results. +If the '-t' (test) option is provided, this tool will attempt to git clone any +project that defines a Bandit job. Once cloned, it will use tox to run the +defined Bandit job and capture logs for any failures. + TODO: Add detection / handling of bandit.yaml for each project. TODO: Deal with different branch definitions in the Zuul layout.yaml. """ +import argparse import datetime +import os import requests +import subprocess import yaml BASE_URL = "https://git.openstack.org/cgit/" -PATH_INFRA = "openstack-infra/project-config/plain/" +GIT_BASE = "https://git.openstack.org/" +PATH_INFRA = "openstack-infra/project-config/plain/" PATH_JENKINS = "jenkins/jobs/projects.yaml" +PATH_PROJECT_LIST = "openstack/governance/plain/reference/projects.yaml" PATH_ZUUL = "zuul/layout.yaml" TITLE = "OpenStack Bandit Coverage Report -- {0} UTC".format( @@ -54,7 +63,7 @@ def get_yaml(url): ) -def coverage_jenkins(conf_jenkins): +def list_projects(conf_jenkins): data = get_yaml("{0}{1}{2}".format(BASE_URL, PATH_INFRA, conf_jenkins)) # parse data bandit_projects = [] @@ -65,6 +74,7 @@ def coverage_jenkins(conf_jenkins): if type(job) == dict and 'gate-{name}-tox-{envlist}' in job: if 'bandit' in job['gate-{name}-tox-{envlist}']['envlist']: bandit_projects.append(project_name) + # output results print("Bandit jobs have been defined in the following OpenStack projects:") for project in sorted(bandit_projects): @@ -72,6 +82,7 @@ def coverage_jenkins(conf_jenkins): print("\n(Configuration from {0}{1}{2})\n".format( BASE_URL, PATH_INFRA, conf_jenkins )) + return bandit_projects def coverage_zuul(conf_zuul): @@ -108,16 +119,158 @@ def coverage_zuul(conf_zuul): )) -def main(): +def _print_title(): print("{0}\n{1}\n{0}\n".format( "=" * len(TITLE), TITLE, "=" * len(TITLE) )) - coverage_jenkins(PATH_JENKINS) + + +def _parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('-t', '--test', dest='do_test', action='store_true', + help='Test upstream project Bandit gates. This will ' + 'clone each upstream project, run Bandit as ' + 'configured in the tox environment, display pass ' + 'status, and save output.') + + parser.set_defaults(do_test=False) + + return parser.parse_args() + + +def _get_repo_names(project_list): + # take a list of project names, like ['anchor', 'barbican'], get the + # corresponding repos for each. Return a dictionary with the project + # as the key and the repo as the value. + project_repos = {key: None for key in project_list} + + yaml_data = get_yaml("{0}{1}".format(BASE_URL, PATH_PROJECT_LIST)) + + for project in yaml_data: + + try: + # if one of the projects we're looking for is listed as a + # deliverable for this project, look for the first listed repo + # for that deliverable + for deliverable in yaml_data[project]['deliverables']: + + if deliverable in project_list: + # the deliverable name is the project we're looking for, + # store the listed repo name for it + project_repos[deliverable] = (yaml_data[project] + ['deliverables'] + [deliverable]['repos'][0]) + + except (KeyError, IndexError): + # improperly formatted entry, keep going + pass + + return project_repos + + +def clone_projects(project_list): + # clone all of the projects, return the directory name they are cloned in + project_locations = _get_repo_names(project_list) + + orig_dir = os.path.abspath(os.getcwd()) + + # create directory for projects + try: + dir_name = 'project-source-{}'.format(datetime.datetime.utcnow(). + strftime('%Y-%m-%d-%H-%M-%S')) + os.mkdir(dir_name) + os.chdir(dir_name) + except OSError: + print("Unable to create directory for cloning projects") + return None + + for project in project_locations: + print '=' * len(TITLE) + print("Cloning project: {} from repo {} into {}". + format(project, project_locations[project], dir_name)) + + try: + subprocess.check_call(['git', 'clone', + GIT_BASE + project_locations[project]]) + + except subprocess.CalledProcessError: + print("Unable to clone project from repo: {}". + format(project_locations[project])) + + os.chdir(orig_dir) + + return os.path.abspath(dir_name) + + +def run_bandit(source_dir): + # go through each source directory in the directory which contains source, + # run Bandit with the established tox job, save results + orig_dir = os.path.abspath(os.getcwd()) + + try: + fail_results_dir = os.path.abspath('fail_results') + os.mkdir(fail_results_dir) + except OSError: + print ("Unable to make results directory") + + os.chdir(source_dir) + + run_success = {} + + for d in os.listdir(os.getcwd()): + os.chdir(d) + + print '=' * len(TITLE) + print 'Running tox Bandit in directory {}'.format(d) + + try: + subprocess.check_output(['tox', '-e', 'bandit'], + stderr=subprocess.STDOUT) + + except subprocess.CalledProcessError as exc: + run_success[d] = False + + # write log containing the process output + fail_log_path = fail_results_dir + '/' + d + with open(fail_log_path, 'w') as f: + f.write(exc.output) + print("Bandit tox failed, wrote failure log to {}". + format(fail_log_path)) + + else: + run_success[d] = True + + os.chdir(source_dir) + + os.chdir(orig_dir) + + return run_success + + +def main(): + _print_title() + + args = _parse_args() + + project_list = list_projects(PATH_JENKINS) coverage_zuul(PATH_ZUUL) print("=" * len(TITLE)) + if args.do_test: + source_dir = clone_projects(project_list) + if source_dir: + results = run_bandit(source_dir) + + # output results table + print "-" * 50 + print "{:40s}{:10s}".format("Project", "Passed") + print "-" * 50 + for project in results: + print "{:40s}{:10s}".format(project, str(results[project])) + if __name__ == "__main__": main()