332 lines
14 KiB
Python
332 lines
14 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
"""
|
|
Runs tempest tests
|
|
|
|
This command is used for running the tempest tests
|
|
|
|
Test Selection
|
|
==============
|
|
Tempest run has several options:
|
|
|
|
* ``--regex/-r``: This is a selection regex like what stestr uses. It will run
|
|
any tests that match on re.match() with the regex
|
|
* ``--smoke/-s``: Run all the tests tagged as smoke
|
|
* ``--black-regex``: It allows to do simple test exclusion via passing a
|
|
rejection/black regexp
|
|
|
|
There are also the ``--blacklist-file`` and ``--whitelist-file`` options that
|
|
let you pass a filepath to tempest run with the file format being a line
|
|
separated regex, with '#' used to signify the start of a comment on a line.
|
|
For example::
|
|
|
|
# Regex file
|
|
^regex1 # Match these tests
|
|
.*regex2 # Match those tests
|
|
|
|
These arguments are just passed into stestr, you can refer to the stestr
|
|
selection docs for more details on how these operate:
|
|
http://stestr.readthedocs.io/en/latest/MANUAL.html#test-selection
|
|
|
|
You can also use the ``--list-tests`` option in conjunction with selection
|
|
arguments to list which tests will be run.
|
|
|
|
You can also use the ``--load-list`` option that lets you pass a filepath to
|
|
tempest run with the file format being in a non-regex format, similar to the
|
|
tests generated by the ``--list-tests`` option. You can specify target tests
|
|
by removing unnecessary tests from a list file which is generated from
|
|
``--list-tests`` option.
|
|
|
|
You can also use ``--worker-file`` option that let you pass a filepath to a
|
|
worker yaml file, allowing you to manually schedule the tests run.
|
|
For example, you can setup a tempest run with
|
|
different concurrences to be used with different regexps.
|
|
An example of worker file is showed below::
|
|
|
|
# YAML Worker file
|
|
- worker:
|
|
# you can have more than one regex per worker
|
|
- tempest.api.*
|
|
- neutron_tempest_tests
|
|
- worker:
|
|
- tempest.scenario.*
|
|
|
|
This will run test matching with 'tempest.api.*' and 'neutron_tempest_tests'
|
|
against worker 1. Run tests matching with 'tempest.scenario.*' under worker 2.
|
|
|
|
You can mix manual scheduling with the standard scheduling mechanisms by
|
|
concurrency field on a worker. For example::
|
|
|
|
# YAML Worker file
|
|
- worker:
|
|
# you can have more than one regex per worker
|
|
- tempest.api.*
|
|
- neutron_tempest_tests
|
|
concurrency: 3
|
|
- worker:
|
|
- tempest.scenario.*
|
|
concurrency: 2
|
|
|
|
This will run tests matching with 'tempest.scenario.*' against 2 workers.
|
|
|
|
This worker file is passed into stestr. For some more details on how it
|
|
operates please refer to the stestr scheduling docs:
|
|
https://stestr.readthedocs.io/en/stable/MANUAL.html#test-scheduling
|
|
|
|
Test Execution
|
|
==============
|
|
There are several options to control how the tests are executed. By default
|
|
tempest will run in parallel with a worker for each CPU present on the machine.
|
|
If you want to adjust the number of workers use the ``--concurrency`` option
|
|
and if you want to run tests serially use ``--serial/-t``
|
|
|
|
Running with Workspaces
|
|
-----------------------
|
|
Tempest run enables you to run your tempest tests from any setup tempest
|
|
workspace it relies on you having setup a tempest workspace with either the
|
|
``tempest init`` or ``tempest workspace`` commands. Then using the
|
|
``--workspace`` CLI option you can specify which one of your workspaces you
|
|
want to run tempest from. Using this option you don't have to run Tempest
|
|
directly with you current working directory being the workspace, Tempest will
|
|
take care of managing everything to be executed from there.
|
|
|
|
Running from Anywhere
|
|
---------------------
|
|
Tempest run provides you with an option to execute tempest from anywhere on
|
|
your system. You are required to provide a config file in this case with the
|
|
``--config-file`` option. When run tempest will create a .stestr
|
|
directory and a .stestr.conf file in your current working directory. This way
|
|
you can use stestr commands directly to inspect the state of the previous run.
|
|
|
|
Test Output
|
|
===========
|
|
By default tempest run's output to STDOUT will be generated using the
|
|
subunit-trace output filter. But, if you would prefer a subunit v2 stream be
|
|
output to STDOUT use the ``--subunit`` flag
|
|
|
|
Combining Runs
|
|
==============
|
|
|
|
There are certain situations in which you want to split a single run of tempest
|
|
across 2 executions of tempest run. (for example to run part of the tests
|
|
serially and others in parallel) To accomplish this but still treat the results
|
|
as a single run you can leverage the ``--combine`` option which will append
|
|
the current run's results with the previous runs.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
|
|
from cliff import command
|
|
from oslo_serialization import jsonutils as json
|
|
from stestr import commands
|
|
|
|
from tempest import clients
|
|
from tempest.cmd import cleanup_service
|
|
from tempest.cmd import init
|
|
from tempest.cmd import workspace
|
|
from tempest.common import credentials_factory as credentials
|
|
from tempest import config
|
|
|
|
CONF = config.CONF
|
|
SAVED_STATE_JSON = "saved_state.json"
|
|
|
|
|
|
class TempestRun(command.Command):
|
|
|
|
def _set_env(self, config_file=None):
|
|
if config_file:
|
|
if os.path.exists(os.path.abspath(config_file)):
|
|
CONF.set_config_path(os.path.abspath(config_file))
|
|
else:
|
|
raise FileNotFoundError(
|
|
"Config file: %s doesn't exist" % config_file)
|
|
|
|
# NOTE(mtreinish): This is needed so that stestr doesn't gobble up any
|
|
# stacktraces on failure.
|
|
if 'TESTR_PDB' in os.environ:
|
|
return
|
|
else:
|
|
os.environ["TESTR_PDB"] = ""
|
|
# NOTE(dims): most of our .stestr.conf try to test for PYTHON
|
|
# environment variable and fall back to "python", under python3
|
|
# if it does not exist. we should set it to the python3 executable
|
|
# to deal with this situation better for now.
|
|
if 'PYTHON' not in os.environ:
|
|
os.environ['PYTHON'] = sys.executable
|
|
|
|
def _create_stestr_conf(self):
|
|
top_level_path = os.path.dirname(os.path.dirname(__file__))
|
|
discover_path = os.path.join(top_level_path, 'test_discover')
|
|
file_contents = init.STESTR_CONF % (discover_path, top_level_path)
|
|
with open('.stestr.conf', 'w+') as stestr_conf_file:
|
|
stestr_conf_file.write(file_contents)
|
|
|
|
def take_action(self, parsed_args):
|
|
if parsed_args.config_file:
|
|
self._set_env(parsed_args.config_file)
|
|
else:
|
|
self._set_env()
|
|
# Workspace execution mode
|
|
if parsed_args.workspace:
|
|
workspace_mgr = workspace.WorkspaceManager(
|
|
parsed_args.workspace_path)
|
|
path = workspace_mgr.get_workspace(parsed_args.workspace)
|
|
if not path:
|
|
sys.exit(
|
|
"The %r workspace isn't registered in "
|
|
"%r. Use 'tempest init' to "
|
|
"register the workspace." %
|
|
(parsed_args.workspace, workspace_mgr.path))
|
|
os.chdir(path)
|
|
if not os.path.isfile('.stestr.conf'):
|
|
self._create_stestr_conf()
|
|
# local execution with config file mode
|
|
elif parsed_args.config_file and not os.path.isfile('.stestr.conf'):
|
|
self._create_stestr_conf()
|
|
elif not os.path.isfile('.stestr.conf'):
|
|
print("No .stestr.conf file was found for local execution")
|
|
sys.exit(2)
|
|
if parsed_args.state:
|
|
self._init_state()
|
|
|
|
regex = self._build_regex(parsed_args)
|
|
return_code = 0
|
|
if parsed_args.list_tests:
|
|
return_code = commands.list_command(
|
|
filters=regex, whitelist_file=parsed_args.whitelist_file,
|
|
blacklist_file=parsed_args.blacklist_file,
|
|
black_regex=parsed_args.black_regex)
|
|
|
|
else:
|
|
serial = not parsed_args.parallel
|
|
return_code = commands.run_command(
|
|
filters=regex, subunit_out=parsed_args.subunit,
|
|
serial=serial, concurrency=parsed_args.concurrency,
|
|
blacklist_file=parsed_args.blacklist_file,
|
|
whitelist_file=parsed_args.whitelist_file,
|
|
black_regex=parsed_args.black_regex,
|
|
worker_path=parsed_args.worker_file,
|
|
load_list=parsed_args.load_list, combine=parsed_args.combine)
|
|
if return_code > 0:
|
|
sys.exit(return_code)
|
|
return return_code
|
|
|
|
def get_description(self):
|
|
return 'Run tempest'
|
|
|
|
def _init_state(self):
|
|
print("Initializing saved state.")
|
|
data = {}
|
|
self.global_services = cleanup_service.get_global_cleanup_services()
|
|
self.admin_mgr = clients.Manager(
|
|
credentials.get_configured_admin_credentials())
|
|
admin_mgr = self.admin_mgr
|
|
kwargs = {'data': data,
|
|
'is_dry_run': False,
|
|
'saved_state_json': data,
|
|
'is_preserve': False,
|
|
'is_save_state': True}
|
|
for service in self.global_services:
|
|
svc = service(admin_mgr, **kwargs)
|
|
svc.run()
|
|
|
|
with open(SAVED_STATE_JSON, 'w+') as f:
|
|
f.write(json.dumps(data, sort_keys=True,
|
|
indent=2, separators=(',', ': ')))
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(TempestRun, self).get_parser(prog_name)
|
|
parser = self._add_args(parser)
|
|
return parser
|
|
|
|
def _add_args(self, parser):
|
|
# workspace args
|
|
parser.add_argument('--workspace', default=None,
|
|
help='Name of tempest workspace to use for running'
|
|
' tests. You can see a list of workspaces '
|
|
'with tempest workspace list')
|
|
parser.add_argument('--workspace-path', default=None,
|
|
dest='workspace_path',
|
|
help="The path to the workspace file, the default "
|
|
"is ~/.tempest/workspace.yaml")
|
|
# Configuration flags
|
|
parser.add_argument('--config-file', default=None, dest='config_file',
|
|
help='Configuration file to run tempest with')
|
|
# test selection args
|
|
regex = parser.add_mutually_exclusive_group()
|
|
regex.add_argument('--smoke', '-s', action='store_true',
|
|
help="Run the smoke tests only")
|
|
regex.add_argument('--regex', '-r', default='',
|
|
help='A normal stestr selection regex used to '
|
|
'specify a subset of tests to run')
|
|
parser.add_argument('--black-regex', dest='black_regex',
|
|
help='A regex to exclude tests that match it')
|
|
parser.add_argument('--whitelist-file', '--whitelist_file',
|
|
help="Path to a whitelist file, this file "
|
|
"contains a separate regex on each "
|
|
"newline.")
|
|
parser.add_argument('--blacklist-file', '--blacklist_file',
|
|
help='Path to a blacklist file, this file '
|
|
'contains a separate regex exclude on '
|
|
'each newline')
|
|
parser.add_argument('--load-list', '--load_list',
|
|
help='Path to a non-regex whitelist file, '
|
|
'this file contains a separate test '
|
|
'on each newline. This command '
|
|
'supports files created by the tempest '
|
|
'run ``--list-tests`` command')
|
|
parser.add_argument('--worker-file', '--worker_file',
|
|
help='Optional path to a worker file. This file '
|
|
'contains each worker configuration to be '
|
|
'used to schedule the tests run')
|
|
# list only args
|
|
parser.add_argument('--list-tests', '-l', action='store_true',
|
|
help='List tests',
|
|
default=False)
|
|
# execution args
|
|
parser.add_argument('--concurrency', '-w',
|
|
type=int, default=0,
|
|
help="The number of workers to use, defaults to "
|
|
"the number of cpus")
|
|
parallel = parser.add_mutually_exclusive_group()
|
|
parallel.add_argument('--parallel', dest='parallel',
|
|
action='store_true',
|
|
help='Run tests in parallel (this is the'
|
|
' default)')
|
|
parallel.add_argument('--serial', '-t', dest='parallel',
|
|
action='store_false',
|
|
help='Run tests serially')
|
|
parser.add_argument('--save-state', dest='state',
|
|
action='store_true',
|
|
help="To save the state of the cloud before "
|
|
"running tempest.")
|
|
# output args
|
|
parser.add_argument("--subunit", action='store_true',
|
|
help='Enable subunit v2 output')
|
|
parser.add_argument("--combine", action='store_true',
|
|
help='Combine the output of this run with the '
|
|
"previous run's as a combined stream in the "
|
|
"stestr repository after it finish")
|
|
|
|
parser.set_defaults(parallel=True)
|
|
return parser
|
|
|
|
def _build_regex(self, parsed_args):
|
|
regex = None
|
|
if parsed_args.smoke:
|
|
regex = ['smoke']
|
|
elif parsed_args.regex:
|
|
regex = parsed_args.regex.split()
|
|
return regex
|