406 lines
18 KiB
Python
406 lines
18 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
Runs tempest tests
|
|
|
|
This command is used for running the tempest tests
|
|
|
|
Test Selection
|
|
==============
|
|
Tempest run has several options:
|
|
|
|
* ``--regex/-r``: This is a selection regex like what stestr uses. It will run
|
|
any tests that match on re.match() with the regex
|
|
* ``--smoke/-s``: Run all the tests tagged as smoke
|
|
* ``--exclude-regex``: It allows to do simple test exclusion via passing a
|
|
rejection/exclude regexp
|
|
|
|
There are also the ``--exclude-list`` and ``--include-list`` options that
|
|
let you pass a filepath to tempest run with the file format being a line
|
|
separated regex, with '#' used to signify the start of a comment on a line.
|
|
For example::
|
|
|
|
# Regex file
|
|
^regex1 # Match these tests
|
|
.*regex2 # Match those tests
|
|
|
|
These arguments are just passed into stestr, you can refer to the stestr
|
|
selection docs for more details on how these operate:
|
|
http://stestr.readthedocs.io/en/latest/MANUAL.html#test-selection
|
|
|
|
You can also use the ``--list-tests`` option in conjunction with selection
|
|
arguments to list which tests will be run.
|
|
|
|
You can also use the ``--load-list`` option that lets you pass a filepath to
|
|
tempest run with the file format being in a non-regex format, similar to the
|
|
tests generated by the ``--list-tests`` option. You can specify target tests
|
|
by removing unnecessary tests from a list file which is generated from
|
|
``--list-tests`` option.
|
|
|
|
You can also use ``--worker-file`` option that let you pass a filepath to a
|
|
worker yaml file, allowing you to manually schedule the tests run.
|
|
For example, you can setup a tempest run with
|
|
different concurrences to be used with different regexps.
|
|
An example of worker file is showed below::
|
|
|
|
# YAML Worker file
|
|
- worker:
|
|
# you can have more than one regex per worker
|
|
- tempest.api.*
|
|
- neutron_tempest_tests
|
|
- worker:
|
|
- tempest.scenario.*
|
|
|
|
This will run test matching with 'tempest.api.*' and 'neutron_tempest_tests'
|
|
against worker 1. Run tests matching with 'tempest.scenario.*' under worker 2.
|
|
|
|
You can mix manual scheduling with the standard scheduling mechanisms by
|
|
concurrency field on a worker. For example::
|
|
|
|
# YAML Worker file
|
|
- worker:
|
|
# you can have more than one regex per worker
|
|
- tempest.api.*
|
|
- neutron_tempest_tests
|
|
concurrency: 3
|
|
- worker:
|
|
- tempest.scenario.*
|
|
concurrency: 2
|
|
|
|
This will run tests matching with 'tempest.scenario.*' against 2 workers.
|
|
|
|
This worker file is passed into stestr. For some more details on how it
|
|
operates please refer to the stestr scheduling docs:
|
|
https://stestr.readthedocs.io/en/stable/MANUAL.html#test-scheduling
|
|
|
|
Test Execution
|
|
==============
|
|
There are several options to control how the tests are executed. By default
|
|
tempest will run in parallel with a worker for each CPU present on the machine.
|
|
If you want to adjust the number of workers use the ``--concurrency`` option
|
|
and if you want to run tests serially use ``--serial/-t``
|
|
|
|
Running with Workspaces
|
|
-----------------------
|
|
Tempest run enables you to run your tempest tests from any setup tempest
|
|
workspace it relies on you having setup a tempest workspace with either the
|
|
``tempest init`` or ``tempest workspace`` commands. Then using the
|
|
``--workspace`` CLI option you can specify which one of your workspaces you
|
|
want to run tempest from. Using this option you don't have to run Tempest
|
|
directly with you current working directory being the workspace, Tempest will
|
|
take care of managing everything to be executed from there.
|
|
|
|
Running from Anywhere
|
|
---------------------
|
|
Tempest run provides you with an option to execute tempest from anywhere on
|
|
your system. You are required to provide a config file in this case with the
|
|
``--config-file`` option. When run tempest will create a .stestr
|
|
directory and a .stestr.conf file in your current working directory. This way
|
|
you can use stestr commands directly to inspect the state of the previous run.
|
|
|
|
Test Output
|
|
===========
|
|
By default tempest run's output to STDOUT will be generated using the
|
|
subunit-trace output filter. But, if you would prefer a subunit v2 stream be
|
|
output to STDOUT use the ``--subunit`` flag
|
|
|
|
Combining Runs
|
|
==============
|
|
|
|
There are certain situations in which you want to split a single run of tempest
|
|
across 2 executions of tempest run. (for example to run part of the tests
|
|
serially and others in parallel) To accomplish this but still treat the results
|
|
as a single run you can leverage the ``--combine`` option which will append
|
|
the current run's results with the previous runs.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
from cliff import command
|
|
from oslo_log import log
|
|
from oslo_serialization import jsonutils as json
|
|
from stestr import commands
|
|
|
|
from tempest import clients
|
|
from tempest.cmd import cleanup_service
|
|
from tempest.cmd import init
|
|
from tempest.cmd import workspace
|
|
from tempest.common import credentials_factory as credentials
|
|
from tempest import config
|
|
|
|
CONF = config.CONF
|
|
SAVED_STATE_JSON = "saved_state.json"
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
class TempestRun(command.Command):
|
|
|
|
def _set_env(self, config_file=None):
|
|
if config_file:
|
|
if os.path.exists(os.path.abspath(config_file)):
|
|
CONF.set_config_path(os.path.abspath(config_file))
|
|
else:
|
|
raise FileNotFoundError(
|
|
"Config file: %s doesn't exist" % config_file)
|
|
|
|
# NOTE(mtreinish): This is needed so that stestr doesn't gobble up any
|
|
# stacktraces on failure.
|
|
if 'TESTR_PDB' in os.environ:
|
|
return
|
|
else:
|
|
os.environ["TESTR_PDB"] = ""
|
|
# NOTE(dims): most of our .stestr.conf try to test for PYTHON
|
|
# environment variable and fall back to "python", under python3
|
|
# if it does not exist. we should set it to the python3 executable
|
|
# to deal with this situation better for now.
|
|
if 'PYTHON' not in os.environ:
|
|
os.environ['PYTHON'] = sys.executable
|
|
|
|
def _create_stestr_conf(self):
|
|
top_level_path = os.path.dirname(os.path.dirname(__file__))
|
|
discover_path = os.path.join(top_level_path, 'test_discover')
|
|
file_contents = init.STESTR_CONF % (discover_path, top_level_path)
|
|
with open('.stestr.conf', 'w+') as stestr_conf_file:
|
|
stestr_conf_file.write(file_contents)
|
|
|
|
def take_action(self, parsed_args):
|
|
if parsed_args.config_file:
|
|
self._set_env(parsed_args.config_file)
|
|
else:
|
|
self._set_env()
|
|
# Workspace execution mode
|
|
if parsed_args.workspace:
|
|
workspace_mgr = workspace.WorkspaceManager(
|
|
parsed_args.workspace_path)
|
|
path = workspace_mgr.get_workspace(parsed_args.workspace)
|
|
if not path:
|
|
sys.exit(
|
|
"The %r workspace isn't registered in "
|
|
"%r. Use 'tempest init' to "
|
|
"register the workspace." %
|
|
(parsed_args.workspace, workspace_mgr.path))
|
|
os.chdir(path)
|
|
if not os.path.isfile('.stestr.conf'):
|
|
self._create_stestr_conf()
|
|
# local execution with config file mode
|
|
elif parsed_args.config_file and not os.path.isfile('.stestr.conf'):
|
|
self._create_stestr_conf()
|
|
elif not os.path.isfile('.stestr.conf'):
|
|
print("No .stestr.conf file was found for local execution")
|
|
sys.exit(2)
|
|
if parsed_args.state:
|
|
self._init_state()
|
|
|
|
regex = self._build_regex(parsed_args)
|
|
|
|
# temporary method for parsing deprecated and new stestr options
|
|
# and showing warning messages in order to make the transition
|
|
# smoother for all tempest consumers
|
|
# TODO(kopecmartin) remove this after stestr>=3.1.0 is used
|
|
# in all supported OpenStack releases
|
|
def parse_dep(old_o, old_v, new_o, new_v):
|
|
ret = ''
|
|
if old_v:
|
|
LOG.warning("'%s' option is deprecated, use '%s' instead "
|
|
"which is functionally equivalent. Right now "
|
|
"Tempest still supports this option for "
|
|
"backward compatibility, however, it will be "
|
|
"removed soon.",
|
|
old_o, new_o)
|
|
ret = old_v
|
|
if old_v and new_v:
|
|
# both options are specified
|
|
LOG.warning("'%s' and '%s' are specified at the same time, "
|
|
"'%s' takes precedence over '%s'",
|
|
new_o, old_o, new_o, old_o)
|
|
if new_v:
|
|
ret = new_v
|
|
return ret
|
|
ex_regex = parse_dep('--black-regex', parsed_args.black_regex,
|
|
'--exclude-regex', parsed_args.exclude_regex)
|
|
ex_list = parse_dep('--blacklist-file', parsed_args.blacklist_file,
|
|
'--exclude-list', parsed_args.exclude_list)
|
|
in_list = parse_dep('--whitelist-file', parsed_args.whitelist_file,
|
|
'--include-list', parsed_args.include_list)
|
|
|
|
return_code = 0
|
|
if parsed_args.list_tests:
|
|
try:
|
|
return_code = commands.list_command(
|
|
filters=regex, include_list=in_list,
|
|
exclude_list=ex_list, exclude_regex=ex_regex)
|
|
except TypeError:
|
|
# exclude_list, include_list and exclude_regex are defined only
|
|
# in stestr >= 3.1.0, this except block catches the case when
|
|
# tempest is executed with an older stestr
|
|
return_code = commands.list_command(
|
|
filters=regex, whitelist_file=in_list,
|
|
blacklist_file=ex_list, black_regex=ex_regex)
|
|
|
|
else:
|
|
serial = not parsed_args.parallel
|
|
params = {
|
|
'filters': regex, 'subunit_out': parsed_args.subunit,
|
|
'serial': serial, 'concurrency': parsed_args.concurrency,
|
|
'worker_path': parsed_args.worker_file,
|
|
'load_list': parsed_args.load_list,
|
|
'combine': parsed_args.combine
|
|
}
|
|
try:
|
|
return_code = commands.run_command(
|
|
**params, exclude_list=ex_list,
|
|
include_list=in_list, exclude_regex=ex_regex)
|
|
except TypeError:
|
|
# exclude_list, include_list and exclude_regex are defined only
|
|
# in stestr >= 3.1.0, this except block catches the case when
|
|
# tempest is executed with an older stestr
|
|
return_code = commands.run_command(
|
|
**params, blacklist_file=ex_list,
|
|
whitelist_file=in_list, black_regex=ex_regex)
|
|
if return_code > 0:
|
|
sys.exit(return_code)
|
|
return return_code
|
|
|
|
def get_description(self):
|
|
return 'Run tempest'
|
|
|
|
def _init_state(self):
|
|
print("Initializing saved state.")
|
|
data = {}
|
|
self.global_services = cleanup_service.get_global_cleanup_services()
|
|
self.admin_mgr = clients.Manager(
|
|
credentials.get_configured_admin_credentials())
|
|
admin_mgr = self.admin_mgr
|
|
kwargs = {'data': data,
|
|
'is_dry_run': False,
|
|
'saved_state_json': data,
|
|
'is_preserve': False,
|
|
'is_save_state': True}
|
|
for service in self.global_services:
|
|
svc = service(admin_mgr, **kwargs)
|
|
svc.run()
|
|
|
|
with open(SAVED_STATE_JSON, 'w+') as f:
|
|
f.write(json.dumps(data, sort_keys=True,
|
|
indent=2, separators=(',', ': ')))
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(TempestRun, self).get_parser(prog_name)
|
|
parser = self._add_args(parser)
|
|
return parser
|
|
|
|
def _add_args(self, parser):
|
|
# workspace args
|
|
parser.add_argument('--workspace', default=None,
|
|
help='Name of tempest workspace to use for running'
|
|
' tests. You can see a list of workspaces '
|
|
'with tempest workspace list')
|
|
parser.add_argument('--workspace-path', default=None,
|
|
dest='workspace_path',
|
|
help="The path to the workspace file, the default "
|
|
"is ~/.tempest/workspace.yaml")
|
|
# Configuration flags
|
|
parser.add_argument('--config-file', default=None, dest='config_file',
|
|
help='Configuration file to run tempest with')
|
|
# test selection args
|
|
regex = parser.add_mutually_exclusive_group()
|
|
regex.add_argument('--smoke', '-s', action='store_true',
|
|
help="Run the smoke tests only")
|
|
regex.add_argument('--regex', '-r', default='',
|
|
help='A normal stestr selection regex used to '
|
|
'specify a subset of tests to run')
|
|
parser.add_argument('--black-regex', dest='black_regex',
|
|
help='DEPRECATED: This option is deprecated and '
|
|
'will be removed soon, use --exclude-regex '
|
|
'which is functionally equivalent. If this '
|
|
'is specified at the same time as '
|
|
'--exclude-regex, this flag will be ignored '
|
|
'and --exclude-regex will be used')
|
|
parser.add_argument('--exclude-regex', dest='exclude_regex',
|
|
help='A regex to exclude tests that match it')
|
|
parser.add_argument('--whitelist-file', '--whitelist_file',
|
|
help='DEPRECATED: This option is deprecated and '
|
|
'will be removed soon, use --include-list '
|
|
'which is functionally equivalent. If this '
|
|
'is specified at the same time as '
|
|
'--include-list, this flag will be ignored '
|
|
'and --include-list will be used')
|
|
parser.add_argument('--include-list', '--include_list',
|
|
help="Path to an include file which contains the "
|
|
"regex for tests to be included in tempest "
|
|
"run, this file contains a separate regex on "
|
|
"each newline.")
|
|
parser.add_argument('--blacklist-file', '--blacklist_file',
|
|
help='DEPRECATED: This option is deprecated and '
|
|
'will be removed soon, use --exclude-list '
|
|
'which is functionally equivalent. If this '
|
|
'is specified at the same time as '
|
|
'--exclude-list, this flag will be ignored '
|
|
'and --exclude-list will be used')
|
|
parser.add_argument('--exclude-list', '--exclude_list',
|
|
help='Path to an exclude file which contains the '
|
|
'regex for tests to be excluded in tempest '
|
|
'run, this file contains a separate regex on '
|
|
'each newline.')
|
|
parser.add_argument('--load-list', '--load_list',
|
|
help='Path to a non-regex whitelist file, '
|
|
'this file contains a separate test '
|
|
'on each newline. This command '
|
|
'supports files created by the tempest '
|
|
'run ``--list-tests`` command')
|
|
parser.add_argument('--worker-file', '--worker_file',
|
|
help='Optional path to a worker file. This file '
|
|
'contains each worker configuration to be '
|
|
'used to schedule the tests run')
|
|
# list only args
|
|
parser.add_argument('--list-tests', '-l', action='store_true',
|
|
help='List tests',
|
|
default=False)
|
|
# execution args
|
|
parser.add_argument('--concurrency', '-w',
|
|
type=int, default=0,
|
|
help="The number of workers to use, defaults to "
|
|
"the number of cpus")
|
|
parallel = parser.add_mutually_exclusive_group()
|
|
parallel.add_argument('--parallel', dest='parallel',
|
|
action='store_true',
|
|
help='Run tests in parallel (this is the'
|
|
' default)')
|
|
parallel.add_argument('--serial', '-t', dest='parallel',
|
|
action='store_false',
|
|
help='Run tests serially')
|
|
parser.add_argument('--save-state', dest='state',
|
|
action='store_true',
|
|
help="To save the state of the cloud before "
|
|
"running tempest.")
|
|
# output args
|
|
parser.add_argument("--subunit", action='store_true',
|
|
help='Enable subunit v2 output')
|
|
parser.add_argument("--combine", action='store_true',
|
|
help='Combine the output of this run with the '
|
|
"previous run's as a combined stream in the "
|
|
"stestr repository after it finish")
|
|
|
|
parser.set_defaults(parallel=True)
|
|
return parser
|
|
|
|
def _build_regex(self, parsed_args):
|
|
regex = None
|
|
if parsed_args.smoke:
|
|
regex = ['smoke']
|
|
elif parsed_args.regex:
|
|
regex = parsed_args.regex.split()
|
|
return regex
|