367 lines
15 KiB
Python
367 lines
15 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
Runs tempest tests
|
|
|
|
This command is used for running the tempest tests
|
|
|
|
Test Selection
|
|
==============
|
|
Tempest run has several options:
|
|
|
|
* ``--regex, -r``: This is a selection regex like what testr uses. It will run
|
|
any tests that match on re.match() with the regex
|
|
* ``--smoke, -s``: Run all the tests tagged as smoke
|
|
|
|
There are also the ``--blacklist-file`` and ``--whitelist-file`` options that
|
|
let you pass a filepath to tempest run with the file format being a line
|
|
separated regex, with '#' used to signify the start of a comment on a line.
|
|
For example::
|
|
|
|
# Regex file
|
|
^regex1 # Match these tests
|
|
.*regex2 # Match those tests
|
|
|
|
The blacklist file will be used to construct a negative lookahead regex and
|
|
the whitelist file will simply OR all the regexes in the file. The whitelist
|
|
and blacklist file options are mutually exclusive so you can't use them
|
|
together. However, you can combine either with a normal regex or the *--smoke*
|
|
flag. When used with a blacklist file the generated regex will be combined to
|
|
something like::
|
|
|
|
^((?!black_regex1|black_regex2).)*$cli_regex1
|
|
|
|
When combined with a whitelist file all the regexes from the file and the CLI
|
|
regexes will be ORed.
|
|
|
|
You can also use the ``--list-tests`` option in conjunction with selection
|
|
arguments to list which tests will be run.
|
|
|
|
You can also use the ``--load-list`` option that lets you pass a filepath to
|
|
tempest run with the file format being in a non-regex format, similar to the
|
|
tests generated by the ``--list-tests`` option. You can specify target tests
|
|
by removing unnecessary tests from a list file which is generated from
|
|
``--list-tests`` option.
|
|
|
|
Test Execution
|
|
==============
|
|
There are several options to control how the tests are executed. By default
|
|
tempest will run in parallel with a worker for each CPU present on the machine.
|
|
If you want to adjust the number of workers use the ``--concurrency`` option
|
|
and if you want to run tests serially use ``--serial/-t``
|
|
|
|
Running with Workspaces
|
|
-----------------------
|
|
Tempest run enables you to run your tempest tests from any setup tempest
|
|
workspace it relies on you having setup a tempest workspace with either the
|
|
``tempest init`` or ``tempest workspace`` commands. Then using the
|
|
``--workspace`` CLI option you can specify which one of your workspaces you
|
|
want to run tempest from. Using this option you don't have to run Tempest
|
|
directly with you current working directory being the workspace, Tempest will
|
|
take care of managing everything to be executed from there.
|
|
|
|
Running from Anywhere
|
|
---------------------
|
|
Tempest run provides you with an option to execute tempest from anywhere on
|
|
your system. You are required to provide a config file in this case with the
|
|
``--config-file`` option. When run tempest will create a .testrepository
|
|
directory and a .testr.conf file in your current working directory. This way
|
|
you can use testr commands directly to inspect the state of the previous run.
|
|
|
|
Test Output
|
|
===========
|
|
By default tempest run's output to STDOUT will be generated using the
|
|
subunit-trace output filter. But, if you would prefer a subunit v2 stream be
|
|
output to STDOUT use the ``--subunit`` flag
|
|
|
|
Combining Runs
|
|
==============
|
|
|
|
There are certain situations in which you want to split a single run of tempest
|
|
across 2 executions of tempest run. (for example to run part of the tests
|
|
serially and others in parallel) To accomplish this but still treat the results
|
|
as a single run you can leverage the ``--combine`` option which will append
|
|
the current run's results with the previous runs.
|
|
"""
|
|
|
|
import io
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
|
|
from cliff import command
|
|
from os_testr import regex_builder
|
|
from os_testr import subunit_trace
|
|
from oslo_serialization import jsonutils as json
|
|
import six
|
|
from testrepository.commands import run_argv
|
|
|
|
from tempest import clients
|
|
from tempest.cmd import cleanup_service
|
|
from tempest.cmd import init
|
|
from tempest.cmd import workspace
|
|
from tempest.common import credentials_factory as credentials
|
|
from tempest import config
|
|
|
|
|
|
CONF = config.CONF
|
|
SAVED_STATE_JSON = "saved_state.json"
|
|
|
|
|
|
class TempestRun(command.Command):
|
|
|
|
def _set_env(self, config_file=None):
|
|
if config_file:
|
|
CONF.set_config_path(os.path.abspath(config_file))
|
|
# NOTE(mtreinish): This is needed so that testr doesn't gobble up any
|
|
# stacktraces on failure.
|
|
if 'TESTR_PDB' in os.environ:
|
|
return
|
|
else:
|
|
os.environ["TESTR_PDB"] = ""
|
|
# NOTE(dims): most of our .testr.conf try to test for PYTHON
|
|
# environment variable and fall back to "python", under python3
|
|
# if it does not exist. we should set it to the python3 executable
|
|
# to deal with this situation better for now.
|
|
if six.PY3 and 'PYTHON' not in os.environ:
|
|
os.environ['PYTHON'] = sys.executable
|
|
|
|
def _create_testrepository(self):
|
|
if not os.path.isdir('.testrepository'):
|
|
returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout,
|
|
sys.stderr)
|
|
if returncode:
|
|
sys.exit(returncode)
|
|
|
|
def _create_testr_conf(self):
|
|
top_level_path = os.path.dirname(os.path.dirname(__file__))
|
|
discover_path = os.path.join(top_level_path, 'test_discover')
|
|
file_contents = init.TESTR_CONF % (top_level_path, discover_path)
|
|
with open('.testr.conf', 'w+') as testr_conf_file:
|
|
testr_conf_file.write(file_contents)
|
|
|
|
def take_action(self, parsed_args):
|
|
returncode = 0
|
|
if parsed_args.config_file:
|
|
self._set_env(parsed_args.config_file)
|
|
else:
|
|
self._set_env()
|
|
# Workspace execution mode
|
|
if parsed_args.workspace:
|
|
workspace_mgr = workspace.WorkspaceManager(
|
|
parsed_args.workspace_path)
|
|
path = workspace_mgr.get_workspace(parsed_args.workspace)
|
|
if not path:
|
|
sys.exit(
|
|
"The %r workspace isn't registered in "
|
|
"%r. Use 'tempest init' to "
|
|
"register the workspace." %
|
|
(parsed_args.workspace, workspace_mgr.path))
|
|
os.chdir(path)
|
|
# NOTE(mtreinish): tempest init should create a .testrepository dir
|
|
# but since workspaces can be imported let's sanity check and
|
|
# ensure that one is created
|
|
self._create_testrepository()
|
|
# Local execution mode
|
|
elif os.path.isfile('.testr.conf'):
|
|
# If you're running in local execution mode and there is not a
|
|
# testrepository dir create one
|
|
self._create_testrepository()
|
|
# local execution with config file mode
|
|
elif parsed_args.config_file:
|
|
self._create_testr_conf()
|
|
self._create_testrepository()
|
|
else:
|
|
print("No .testr.conf file was found for local execution")
|
|
sys.exit(2)
|
|
if parsed_args.state:
|
|
self._init_state()
|
|
else:
|
|
pass
|
|
|
|
if parsed_args.combine:
|
|
temp_stream = tempfile.NamedTemporaryFile()
|
|
return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin,
|
|
temp_stream, sys.stderr)
|
|
if return_code > 0:
|
|
sys.exit(return_code)
|
|
|
|
regex = self._build_regex(parsed_args)
|
|
if parsed_args.list_tests:
|
|
argv = ['tempest', 'list-tests', regex]
|
|
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
|
|
else:
|
|
options = self._build_options(parsed_args)
|
|
returncode = self._run(regex, options)
|
|
if returncode > 0:
|
|
sys.exit(returncode)
|
|
|
|
if parsed_args.combine:
|
|
return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin,
|
|
temp_stream, sys.stderr)
|
|
if return_code > 0:
|
|
sys.exit(return_code)
|
|
returncode = run_argv(['tempest', 'load', temp_stream.name],
|
|
sys.stdin, sys.stdout, sys.stderr)
|
|
sys.exit(returncode)
|
|
|
|
def get_description(self):
|
|
return 'Run tempest'
|
|
|
|
def _init_state(self):
|
|
print("Initializing saved state.")
|
|
data = {}
|
|
self.global_services = cleanup_service.get_global_cleanup_services()
|
|
self.admin_mgr = clients.Manager(
|
|
credentials.get_configured_admin_credentials())
|
|
admin_mgr = self.admin_mgr
|
|
kwargs = {'data': data,
|
|
'is_dry_run': False,
|
|
'saved_state_json': data,
|
|
'is_preserve': False,
|
|
'is_save_state': True}
|
|
for service in self.global_services:
|
|
svc = service(admin_mgr, **kwargs)
|
|
svc.run()
|
|
|
|
with open(SAVED_STATE_JSON, 'w+') as f:
|
|
f.write(json.dumps(data,
|
|
sort_keys=True, indent=2, separators=(',', ': ')))
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(TempestRun, self).get_parser(prog_name)
|
|
parser = self._add_args(parser)
|
|
return parser
|
|
|
|
def _add_args(self, parser):
|
|
# workspace args
|
|
parser.add_argument('--workspace', default=None,
|
|
help='Name of tempest workspace to use for running'
|
|
' tests. You can see a list of workspaces '
|
|
'with tempest workspace list')
|
|
parser.add_argument('--workspace-path', default=None,
|
|
dest='workspace_path',
|
|
help="The path to the workspace file, the default "
|
|
"is ~/.tempest/workspace.yaml")
|
|
# Configuration flags
|
|
parser.add_argument('--config-file', default=None, dest='config_file',
|
|
help='Configuration file to run tempest with')
|
|
# test selection args
|
|
regex = parser.add_mutually_exclusive_group()
|
|
regex.add_argument('--smoke', '-s', action='store_true',
|
|
help="Run the smoke tests only")
|
|
regex.add_argument('--regex', '-r', default='',
|
|
help='A normal testr selection regex used to '
|
|
'specify a subset of tests to run')
|
|
list_selector = parser.add_mutually_exclusive_group()
|
|
list_selector.add_argument('--whitelist-file', '--whitelist_file',
|
|
help="Path to a whitelist file, this file "
|
|
"contains a separate regex on each "
|
|
"newline.")
|
|
list_selector.add_argument('--blacklist-file', '--blacklist_file',
|
|
help='Path to a blacklist file, this file '
|
|
'contains a separate regex exclude on '
|
|
'each newline')
|
|
list_selector.add_argument('--load-list', '--load_list',
|
|
help='Path to a non-regex whitelist file, '
|
|
'this file contains a seperate test '
|
|
'on each newline. This command'
|
|
'supports files created by the tempest'
|
|
'run ``--list-tests`` command')
|
|
# list only args
|
|
parser.add_argument('--list-tests', '-l', action='store_true',
|
|
help='List tests',
|
|
default=False)
|
|
# execution args
|
|
parser.add_argument('--concurrency', '-w',
|
|
help="The number of workers to use, defaults to "
|
|
"the number of cpus")
|
|
parallel = parser.add_mutually_exclusive_group()
|
|
parallel.add_argument('--parallel', dest='parallel',
|
|
action='store_true',
|
|
help='Run tests in parallel (this is the'
|
|
' default)')
|
|
parallel.add_argument('--serial', '-t', dest='parallel',
|
|
action='store_false',
|
|
help='Run tests serially')
|
|
parser.add_argument('--save-state', dest='state',
|
|
action='store_true',
|
|
help="To save the state of the cloud before "
|
|
"running tempest.")
|
|
# output args
|
|
parser.add_argument("--subunit", action='store_true',
|
|
help='Enable subunit v2 output')
|
|
parser.add_argument("--combine", action='store_true',
|
|
help='Combine the output of this run with the '
|
|
"previous run's as a combined stream in the "
|
|
"testr repository after it finish")
|
|
|
|
parser.set_defaults(parallel=True)
|
|
return parser
|
|
|
|
def _build_regex(self, parsed_args):
|
|
regex = ''
|
|
if parsed_args.smoke:
|
|
regex = 'smoke'
|
|
elif parsed_args.regex:
|
|
regex = parsed_args.regex
|
|
if parsed_args.whitelist_file or parsed_args.blacklist_file:
|
|
regex = regex_builder.construct_regex(parsed_args.blacklist_file,
|
|
parsed_args.whitelist_file,
|
|
regex, False)
|
|
return regex
|
|
|
|
def _build_options(self, parsed_args):
|
|
options = []
|
|
if parsed_args.subunit:
|
|
options.append("--subunit")
|
|
if parsed_args.parallel:
|
|
options.append("--parallel")
|
|
if parsed_args.concurrency:
|
|
options.append("--concurrency=%s" % parsed_args.concurrency)
|
|
if parsed_args.load_list:
|
|
options.append("--load-list=%s" % parsed_args.load_list)
|
|
return options
|
|
|
|
def _run(self, regex, options):
|
|
returncode = 0
|
|
argv = ['tempest', 'run', regex] + options
|
|
if '--subunit' in options:
|
|
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
|
|
else:
|
|
argv.append('--subunit')
|
|
stdin = io.StringIO()
|
|
stdout_r, stdout_w = os.pipe()
|
|
subunit_w = os.fdopen(stdout_w, 'wt')
|
|
subunit_r = os.fdopen(stdout_r)
|
|
returncodes = {}
|
|
|
|
def run_argv_thread():
|
|
returncodes['testr'] = run_argv(argv, stdin, subunit_w,
|
|
sys.stderr)
|
|
subunit_w.close()
|
|
|
|
run_thread = threading.Thread(target=run_argv_thread)
|
|
run_thread.start()
|
|
returncodes['subunit-trace'] = subunit_trace.trace(
|
|
subunit_r, sys.stdout, post_fails=True, print_failures=True)
|
|
run_thread.join()
|
|
subunit_r.close()
|
|
# python version of pipefail
|
|
if returncodes['testr']:
|
|
returncode = returncodes['testr']
|
|
elif returncodes['subunit-trace']:
|
|
returncode = returncodes['subunit-trace']
|
|
return returncode
|