browbeat/browbeat.py
Sai Sindhur Malleni 665d0ef715 Some fixes to make log info consistent with change
After code was added to mix-match workload, the stats of executed,
passed and failed tests kept by each workload class have become
inconsistent. It makes sense to only maintain a count at the total
number of tests executed/passed/failed vs per  workload.

Change-Id: I4f84f4580ac29206e7ce660222a2a396e419cac8
2018-01-10 20:35:19 -05:00

172 lines
7.1 KiB
Python
Executable File

#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import datetime
import logging
import os
import sys
import time
import browbeat.elastic
import browbeat.tools
import browbeat.workloadbase
from browbeat.config import load_browbeat_config
from browbeat.path import results_path
_workload_opts = ['perfkit', 'rally', 'shaker', 'yoda']
_config_file = 'browbeat-config.yaml'
debug_log_file = 'log/debug.log'
def main():
parser = argparse.ArgumentParser(
description="Browbeat Performance and Scale testing for Openstack")
parser.add_argument(
'-s', '--setup', nargs='?', default=_config_file,
help='Provide Browbeat YAML configuration file. Default is ./{}'.format(_config_file))
parser.add_argument(
'workloads', nargs='*', help='Browbeat workload(s). Takes a space separated'
' list of workloads ({}) or \"all\"'.format(', '.join(_workload_opts)))
parser.add_argument('--debug', action='store_true', help='Enable Debug messages')
parser.add_argument(
'-p', '--postprocess', dest="path", help="Path to process, ie results/20171130-191420/")
parser.add_argument(
'-c', '--compare', help="Compare metadata", dest="compare", choices=['software-metadata'])
parser.add_argument('-u', '--uuid', help="UUIDs to pass", dest="uuids", nargs=2)
_cli_args = parser.parse_args()
_logger = logging.getLogger('browbeat')
_logger.setLevel(logging.DEBUG)
_formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)7s - %(message)s')
_formatter.converter = time.gmtime
_dbg_file = logging.FileHandler(debug_log_file)
_dbg_file.setLevel(logging.DEBUG)
_dbg_file.setFormatter(_formatter)
_ch = logging.StreamHandler(stream=sys.stdout)
if _cli_args.debug:
_ch.setLevel(logging.DEBUG)
else:
_ch.setLevel(logging.INFO)
_ch.setFormatter(_formatter)
_logger.addHandler(_dbg_file)
_logger.addHandler(_ch)
_logger.debug("CLI Args: {}".format(_cli_args))
# Load Browbeat yaml config file:
_config = load_browbeat_config(_cli_args.setup)
tools = browbeat.tools.Tools(_config)
# Browbeat compare
if _cli_args.compare == "software-metadata":
es = browbeat.elastic.Elastic(_config, "BrowbeatCLI")
es.compare_metadata("_all", 'controller', _cli_args.uuids)
exit(0)
if _cli_args.compare:
parser.print_help()
exit(1)
# Browbeat postprocess
if _cli_args.path:
_logger.info("Browbeat Postprocessing {}".format(_cli_args.path))
return tools.post_process(_cli_args)
# Browbeat workload - "browbeat run"
if _cli_args.workloads == []:
_cli_args.workloads.append('all')
if len(_cli_args.workloads) == 1 and 'all' in _cli_args.workloads:
_cli_args.workloads = _workload_opts
invalid_wkld = [wkld for wkld in _cli_args.workloads if wkld not in _workload_opts]
if invalid_wkld:
_logger.error("Invalid workload(s) specified: {}".format(invalid_wkld))
if 'all' in _cli_args.workloads:
_logger.error(
"If you meant 'all' use: './browbeat.py all' or './browbeat.py'")
exit(1)
result_dir_ts = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S")
_logger.info("Browbeat test suite kicked off")
_logger.info("Browbeat UUID: {}".format(browbeat.elastic.browbeat_uuid))
if _config['elasticsearch']['enabled']:
_logger.info("Checking for Metadata")
metadata_exists = tools.check_metadata()
if not metadata_exists:
_logger.error("Elasticsearch has been enabled but"
" metadata files do not exist")
_logger.info("Gathering Metadata")
tools.gather_metadata()
elif _config['elasticsearch']['regather']:
_logger.info("Regathering Metadata")
tools.gather_metadata()
_logger.info("Running workload(s): {}".format(','.join(_cli_args.workloads)))
# Iteration rerun_type pushes rerun logic down to the workload itself. This allows the workload
# to run multiple times before moving to the next workload
if _config["browbeat"]["rerun_type"] == "iteration":
for workload in _config["workloads"]:
if not workload["enabled"]:
_logger.info("{} workload {} disabled in browbeat config".format(workload["type"],
workload["name"]))
continue
if not workload["type"] in _cli_args.workloads:
_logger.info(
"{} workload {} disabled via cli".format(workload["type"], workload["name"]))
continue
_logger.info("{} workload {} is enabled".format(workload["type"], workload["name"]))
tools.run_workload(workload, result_dir_ts, 0)
browbeat.workloadbase.WorkloadBase.display_summary()
elif _config["browbeat"]["rerun_type"] == "complete":
# Complete rerun_type, reruns after all workloads have been run.
for run_iteration in range(0, _config["browbeat"]["rerun"]):
for workload in _config["workloads"]:
if not workload["enabled"]:
_logger.info("{} workload {} disabled in browbeat config"
.format(workload["type"], workload["name"]))
continue
if not workload["type"] in _cli_args.workloads:
_logger.info("{} workload {} disabled via cli".format(workload["type"],
workload["name"]))
continue
_logger.info("{} workload {} is enabled".format(workload["type"], workload["name"]))
tools.run_workload(workload, result_dir_ts, run_iteration)
browbeat.workloadbase.WorkloadBase.display_summary()
browbeat.workloadbase.WorkloadBase.dump_report(results_path, result_dir_ts)
_logger.info("Saved browbeat result summary to {}"
.format(os.path.join(results_path, "{}.report".format(result_dir_ts))))
if browbeat.workloadbase.WorkloadBase.failure > 0:
_logger.info(
"Browbeat finished with test failures, UUID: {}".format(browbeat.elastic.browbeat_uuid))
sys.exit(1)
if browbeat.workloadbase.WorkloadBase.index_failures > 0:
_logger.info("Browbeat finished with Elasticsearch indexing failures, UUID: {}"
.format(browbeat.elastic.browbeat_uuid))
sys.exit(2)
_logger.info("Browbeat finished successfully, UUID: {}".format(browbeat.elastic.browbeat_uuid))
sys.exit(0)
if __name__ == '__main__':
sys.exit(main())