Adding test tool for check OpenStack projects' Bandit job
This commit adds to the openstack coverage tool. Specifically we add a '-t' option. If provided, this option will git clone any project which uses Bandit in a job, run the tox Bandit job, capture the output of any failed run, and display a summary table at the end. This tool is to support pre-release Bandit checking to make sure that our changes haven't introduced any new issues in projects which use it. Change-Id: I321bcb15b59e3ee00ed2f2c6c2c890b77f30370e
This commit is contained in:
Travis McPeak
parent
7ca6335158
commit
215fb64143
|
|
@ -21,19 +21,28 @@ within the openstack-infra/project-config repository.
|
|||
Parses out Bandit jobs and tests as defined within these configurations.
|
||||
Prints the summary of results.
|
||||
|
||||
If the '-t' (test) option is provided, this tool will attempt to git clone any
|
||||
project that defines a Bandit job. Once cloned, it will use tox to run the
|
||||
defined Bandit job and capture logs for any failures.
|
||||
|
||||
TODO: Add detection / handling of bandit.yaml for each project.
|
||||
TODO: Deal with different branch definitions in the Zuul layout.yaml.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import os
|
||||
import requests
|
||||
import subprocess
|
||||
import yaml
|
||||
|
||||
|
||||
BASE_URL = "https://git.openstack.org/cgit/"
|
||||
PATH_INFRA = "openstack-infra/project-config/plain/"
|
||||
GIT_BASE = "https://git.openstack.org/"
|
||||
|
||||
PATH_INFRA = "openstack-infra/project-config/plain/"
|
||||
PATH_JENKINS = "jenkins/jobs/projects.yaml"
|
||||
PATH_PROJECT_LIST = "openstack/governance/plain/reference/projects.yaml"
|
||||
PATH_ZUUL = "zuul/layout.yaml"
|
||||
|
||||
TITLE = "OpenStack Bandit Coverage Report -- {0} UTC".format(
|
||||
|
|
@ -54,7 +63,7 @@ def get_yaml(url):
|
|||
)
|
||||
|
||||
|
||||
def coverage_jenkins(conf_jenkins):
|
||||
def list_projects(conf_jenkins):
|
||||
data = get_yaml("{0}{1}{2}".format(BASE_URL, PATH_INFRA, conf_jenkins))
|
||||
# parse data
|
||||
bandit_projects = []
|
||||
|
|
@ -65,6 +74,7 @@ def coverage_jenkins(conf_jenkins):
|
|||
if type(job) == dict and 'gate-{name}-tox-{envlist}' in job:
|
||||
if 'bandit' in job['gate-{name}-tox-{envlist}']['envlist']:
|
||||
bandit_projects.append(project_name)
|
||||
|
||||
# output results
|
||||
print("Bandit jobs have been defined in the following OpenStack projects:")
|
||||
for project in sorted(bandit_projects):
|
||||
|
|
@ -72,6 +82,7 @@ def coverage_jenkins(conf_jenkins):
|
|||
print("\n(Configuration from {0}{1}{2})\n".format(
|
||||
BASE_URL, PATH_INFRA, conf_jenkins
|
||||
))
|
||||
return bandit_projects
|
||||
|
||||
|
||||
def coverage_zuul(conf_zuul):
|
||||
|
|
@ -108,16 +119,158 @@ def coverage_zuul(conf_zuul):
|
|||
))
|
||||
|
||||
|
||||
def main():
|
||||
def _print_title():
|
||||
print("{0}\n{1}\n{0}\n".format(
|
||||
"=" * len(TITLE),
|
||||
TITLE,
|
||||
"=" * len(TITLE)
|
||||
))
|
||||
coverage_jenkins(PATH_JENKINS)
|
||||
|
||||
|
||||
def _parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument('-t', '--test', dest='do_test', action='store_true',
|
||||
help='Test upstream project Bandit gates. This will '
|
||||
'clone each upstream project, run Bandit as '
|
||||
'configured in the tox environment, display pass '
|
||||
'status, and save output.')
|
||||
|
||||
parser.set_defaults(do_test=False)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _get_repo_names(project_list):
|
||||
# take a list of project names, like ['anchor', 'barbican'], get the
|
||||
# corresponding repos for each. Return a dictionary with the project
|
||||
# as the key and the repo as the value.
|
||||
project_repos = {key: None for key in project_list}
|
||||
|
||||
yaml_data = get_yaml("{0}{1}".format(BASE_URL, PATH_PROJECT_LIST))
|
||||
|
||||
for project in yaml_data:
|
||||
|
||||
try:
|
||||
# if one of the projects we're looking for is listed as a
|
||||
# deliverable for this project, look for the first listed repo
|
||||
# for that deliverable
|
||||
for deliverable in yaml_data[project]['deliverables']:
|
||||
|
||||
if deliverable in project_list:
|
||||
# the deliverable name is the project we're looking for,
|
||||
# store the listed repo name for it
|
||||
project_repos[deliverable] = (yaml_data[project]
|
||||
['deliverables']
|
||||
[deliverable]['repos'][0])
|
||||
|
||||
except (KeyError, IndexError):
|
||||
# improperly formatted entry, keep going
|
||||
pass
|
||||
|
||||
return project_repos
|
||||
|
||||
|
||||
def clone_projects(project_list):
|
||||
# clone all of the projects, return the directory name they are cloned in
|
||||
project_locations = _get_repo_names(project_list)
|
||||
|
||||
orig_dir = os.path.abspath(os.getcwd())
|
||||
|
||||
# create directory for projects
|
||||
try:
|
||||
dir_name = 'project-source-{}'.format(datetime.datetime.utcnow().
|
||||
strftime('%Y-%m-%d-%H-%M-%S'))
|
||||
os.mkdir(dir_name)
|
||||
os.chdir(dir_name)
|
||||
except OSError:
|
||||
print("Unable to create directory for cloning projects")
|
||||
return None
|
||||
|
||||
for project in project_locations:
|
||||
print '=' * len(TITLE)
|
||||
print("Cloning project: {} from repo {} into {}".
|
||||
format(project, project_locations[project], dir_name))
|
||||
|
||||
try:
|
||||
subprocess.check_call(['git', 'clone',
|
||||
GIT_BASE + project_locations[project]])
|
||||
|
||||
except subprocess.CalledProcessError:
|
||||
print("Unable to clone project from repo: {}".
|
||||
format(project_locations[project]))
|
||||
|
||||
os.chdir(orig_dir)
|
||||
|
||||
return os.path.abspath(dir_name)
|
||||
|
||||
|
||||
def run_bandit(source_dir):
|
||||
# go through each source directory in the directory which contains source,
|
||||
# run Bandit with the established tox job, save results
|
||||
orig_dir = os.path.abspath(os.getcwd())
|
||||
|
||||
try:
|
||||
fail_results_dir = os.path.abspath('fail_results')
|
||||
os.mkdir(fail_results_dir)
|
||||
except OSError:
|
||||
print ("Unable to make results directory")
|
||||
|
||||
os.chdir(source_dir)
|
||||
|
||||
run_success = {}
|
||||
|
||||
for d in os.listdir(os.getcwd()):
|
||||
os.chdir(d)
|
||||
|
||||
print '=' * len(TITLE)
|
||||
print 'Running tox Bandit in directory {}'.format(d)
|
||||
|
||||
try:
|
||||
subprocess.check_output(['tox', '-e', 'bandit'],
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
except subprocess.CalledProcessError as exc:
|
||||
run_success[d] = False
|
||||
|
||||
# write log containing the process output
|
||||
fail_log_path = fail_results_dir + '/' + d
|
||||
with open(fail_log_path, 'w') as f:
|
||||
f.write(exc.output)
|
||||
print("Bandit tox failed, wrote failure log to {}".
|
||||
format(fail_log_path))
|
||||
|
||||
else:
|
||||
run_success[d] = True
|
||||
|
||||
os.chdir(source_dir)
|
||||
|
||||
os.chdir(orig_dir)
|
||||
|
||||
return run_success
|
||||
|
||||
|
||||
def main():
|
||||
_print_title()
|
||||
|
||||
args = _parse_args()
|
||||
|
||||
project_list = list_projects(PATH_JENKINS)
|
||||
coverage_zuul(PATH_ZUUL)
|
||||
print("=" * len(TITLE))
|
||||
|
||||
if args.do_test:
|
||||
source_dir = clone_projects(project_list)
|
||||
if source_dir:
|
||||
results = run_bandit(source_dir)
|
||||
|
||||
# output results table
|
||||
print "-" * 50
|
||||
print "{:40s}{:10s}".format("Project", "Passed")
|
||||
print "-" * 50
|
||||
for project in results:
|
||||
print "{:40s}{:10s}".format(project, str(results[project]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue