Merge "Add tempest run command"
This commit is contained in:
commit
5871b197c7
@ -52,6 +52,7 @@ Command Documentation
|
||||
cleanup
|
||||
javelin
|
||||
workspace
|
||||
run
|
||||
|
||||
==================
|
||||
Indices and tables
|
||||
|
5
doc/source/run.rst
Normal file
5
doc/source/run.rst
Normal file
@ -0,0 +1,5 @@
|
||||
-----------
|
||||
Tempest Run
|
||||
-----------
|
||||
|
||||
.. automodule:: tempest.cmd.run
|
4
releasenotes/notes/add-tempest-run-3d0aaf69c2ca4115.yaml
Normal file
4
releasenotes/notes/add-tempest-run-3d0aaf69c2ca4115.yaml
Normal file
@ -0,0 +1,4 @@
|
||||
---
|
||||
features:
|
||||
- Adds the tempest run command to the unified tempest CLI. This new command
|
||||
is used for running tempest tests.
|
@ -42,6 +42,7 @@ tempest.cm =
|
||||
list-plugins = tempest.cmd.list_plugins:TempestListPlugins
|
||||
verify-config = tempest.cmd.verify_tempest_config:TempestVerifyConfig
|
||||
workspace = tempest.cmd.workspace:TempestWorkspace
|
||||
run = tempest.cmd.run:TempestRun
|
||||
oslo.config.opts =
|
||||
tempest.config = tempest.config:list_opts
|
||||
|
||||
|
181
tempest/cmd/run.py
Normal file
181
tempest/cmd/run.py
Normal file
@ -0,0 +1,181 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Runs tempest tests
|
||||
|
||||
This command is used for running the tempest tests
|
||||
|
||||
Test Selection
|
||||
==============
|
||||
Tempest run has several options:
|
||||
|
||||
* **--regex/-r**: This is a selection regex like what testr uses. It will run
|
||||
any tests that match on re.match() with the regex
|
||||
* **--smoke**: Run all the tests tagged as smoke
|
||||
|
||||
You can also use the **--list-tests** option in conjunction with selection
|
||||
arguments to list which tests will be run.
|
||||
|
||||
Test Execution
|
||||
==============
|
||||
There are several options to control how the tests are executed. By default
|
||||
tempest will run in parallel with a worker for each CPU present on the machine.
|
||||
If you want to adjust the number of workers use the **--concurrency** option
|
||||
and if you want to run tests serially use **--serial**
|
||||
|
||||
Test Output
|
||||
===========
|
||||
By default tempest run's output to STDOUT will be generated using the
|
||||
subunit-trace output filter. But, if you would prefer a subunit v2 stream be
|
||||
output to STDOUT use the **--subunit** flag
|
||||
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
|
||||
from cliff import command
|
||||
from os_testr import subunit_trace
|
||||
from oslo_log import log as logging
|
||||
from testrepository.commands import run_argv
|
||||
|
||||
from tempest import config
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = config.CONF
|
||||
|
||||
|
||||
class TempestRun(command.Command):
|
||||
|
||||
def _set_env(self):
|
||||
# NOTE(mtreinish): This is needed so that testr doesn't gobble up any
|
||||
# stacktraces on failure.
|
||||
if 'TESTR_PDB' in os.environ:
|
||||
return
|
||||
else:
|
||||
os.environ["TESTR_PDB"] = ""
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
self._set_env()
|
||||
# Local exceution mode
|
||||
if os.path.isfile('.testr.conf'):
|
||||
# If you're running in local execution mode and there is not a
|
||||
# testrepository dir create one
|
||||
if not os.path.isdir('.testrepository'):
|
||||
returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout,
|
||||
sys.stderr)
|
||||
if returncode:
|
||||
sys.exit(returncode)
|
||||
else:
|
||||
print("No .testr.conf file was found for local exceution")
|
||||
sys.exit(2)
|
||||
|
||||
regex = self._build_regex(parsed_args)
|
||||
if parsed_args.list_tests:
|
||||
argv = ['tempest', 'list-tests', regex]
|
||||
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
|
||||
else:
|
||||
options = self._build_options(parsed_args)
|
||||
returncode = self._run(regex, options)
|
||||
sys.exit(returncode)
|
||||
|
||||
def get_description(self):
|
||||
return 'Run tempest'
|
||||
|
||||
def get_parser(self, prog_name):
|
||||
parser = super(TempestRun, self).get_parser(prog_name)
|
||||
parser = self._add_args(parser)
|
||||
return parser
|
||||
|
||||
def _add_args(self, parser):
|
||||
# test selection args
|
||||
regex = parser.add_mutually_exclusive_group()
|
||||
regex.add_argument('--smoke', action='store_true',
|
||||
help="Run the smoke tests only")
|
||||
regex.add_argument('--regex', '-r', default='',
|
||||
help='A normal testr selection regex used to '
|
||||
'specify a subset of tests to run')
|
||||
# list only args
|
||||
parser.add_argument('--list-tests', '-l', action='store_true',
|
||||
help='List tests',
|
||||
default=False)
|
||||
# exectution args
|
||||
parser.add_argument('--concurrency', '-w',
|
||||
help="The number of workers to use, defaults to "
|
||||
"the number of cpus")
|
||||
parallel = parser.add_mutually_exclusive_group()
|
||||
parallel.add_argument('--parallel', dest='parallel',
|
||||
action='store_true',
|
||||
help='Run tests in parallel (this is the'
|
||||
' default)')
|
||||
parallel.add_argument('--serial', dest='parallel',
|
||||
action='store_false',
|
||||
help='Run tests serially')
|
||||
# output args
|
||||
parser.add_argument("--subunit", action='store_true',
|
||||
help='Enable subunit v2 output')
|
||||
|
||||
parser.set_defaults(parallel=True)
|
||||
return parser
|
||||
|
||||
def _build_regex(self, parsed_args):
|
||||
regex = ''
|
||||
if parsed_args.smoke:
|
||||
regex = 'smoke'
|
||||
elif parsed_args.regex:
|
||||
regex = parsed_args.regex
|
||||
return regex
|
||||
|
||||
def _build_options(self, parsed_args):
|
||||
options = []
|
||||
if parsed_args.subunit:
|
||||
options.append("--subunit")
|
||||
if parsed_args.parallel:
|
||||
options.append("--parallel")
|
||||
if parsed_args.concurrency:
|
||||
options.append("--concurrency=%s" % parsed_args.concurrency)
|
||||
return options
|
||||
|
||||
def _run(self, regex, options):
|
||||
returncode = 0
|
||||
argv = ['tempest', 'run', regex] + options
|
||||
if '--subunit' in options:
|
||||
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
|
||||
else:
|
||||
argv.append('--subunit')
|
||||
stdin = io.StringIO()
|
||||
stdout_r, stdout_w = os.pipe()
|
||||
subunit_w = os.fdopen(stdout_w, 'wt')
|
||||
subunit_r = os.fdopen(stdout_r)
|
||||
returncodes = {}
|
||||
|
||||
def run_argv_thread():
|
||||
returncodes['testr'] = run_argv(argv, stdin, subunit_w,
|
||||
sys.stderr)
|
||||
subunit_w.close()
|
||||
|
||||
run_thread = threading.Thread(target=run_argv_thread)
|
||||
run_thread.start()
|
||||
returncodes['subunit-trace'] = subunit_trace.trace(subunit_r,
|
||||
sys.stdout)
|
||||
run_thread.join()
|
||||
subunit_r.close()
|
||||
# python version of pipefail
|
||||
if returncodes['testr']:
|
||||
returncode = returncodes['testr']
|
||||
elif returncodes['subunit-trace']:
|
||||
returncode = returncodes['subunit-trace']
|
||||
return returncode
|
110
tempest/tests/cmd/test_run.py
Normal file
110
tempest/tests/cmd/test_run.py
Normal file
@ -0,0 +1,110 @@
|
||||
# Copyright 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
import mock
|
||||
|
||||
from tempest.cmd import run
|
||||
from tempest.tests import base
|
||||
|
||||
DEVNULL = open(os.devnull, 'wb')
|
||||
|
||||
|
||||
class TestTempestRun(base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestTempestRun, self).setUp()
|
||||
self.run_cmd = run.TempestRun(None, None)
|
||||
|
||||
def test_build_options(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, "subunit", True)
|
||||
setattr(args, "parallel", False)
|
||||
setattr(args, "concurrency", 10)
|
||||
options = self.run_cmd._build_options(args)
|
||||
self.assertEqual(['--subunit',
|
||||
'--concurrency=10'],
|
||||
options)
|
||||
|
||||
def test__build_regex_default(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, 'smoke', False)
|
||||
setattr(args, 'regex', '')
|
||||
self.assertEqual('', self.run_cmd._build_regex(args))
|
||||
|
||||
def test__build_regex_smoke(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, "smoke", True)
|
||||
setattr(args, 'regex', '')
|
||||
self.assertEqual('smoke', self.run_cmd._build_regex(args))
|
||||
|
||||
def test__build_regex_regex(self):
|
||||
args = mock.Mock(spec=argparse.Namespace)
|
||||
setattr(args, 'smoke', False)
|
||||
setattr(args, "regex", 'i_am_a_fun_little_regex')
|
||||
self.assertEqual('i_am_a_fun_little_regex',
|
||||
self.run_cmd._build_regex(args))
|
||||
|
||||
|
||||
class TestRunReturnCode(base.TestCase):
|
||||
def setUp(self):
|
||||
super(TestRunReturnCode, self).setUp()
|
||||
# Setup test dirs
|
||||
self.directory = tempfile.mkdtemp(prefix='tempest-unit')
|
||||
self.addCleanup(shutil.rmtree, self.directory)
|
||||
self.test_dir = os.path.join(self.directory, 'tests')
|
||||
os.mkdir(self.test_dir)
|
||||
# Setup Test files
|
||||
self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
|
||||
self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
|
||||
self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
|
||||
self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
|
||||
self.init_file = os.path.join(self.test_dir, '__init__.py')
|
||||
self.setup_py = os.path.join(self.directory, 'setup.py')
|
||||
shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
|
||||
shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
|
||||
shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
|
||||
shutil.copy('setup.py', self.setup_py)
|
||||
shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file)
|
||||
shutil.copy('tempest/tests/files/__init__.py', self.init_file)
|
||||
# Change directory, run wrapper and check result
|
||||
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
|
||||
os.chdir(self.directory)
|
||||
|
||||
def assertRunExit(self, cmd, expected):
|
||||
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
out, err = p.communicate()
|
||||
msg = ("Running %s got an unexpected returncode\n"
|
||||
"Stdout: %s\nStderr: %s" % (' '.join(cmd), out, err))
|
||||
self.assertEqual(p.returncode, expected, msg)
|
||||
|
||||
def test_tempest_run_passes(self):
|
||||
# Git init is required for the pbr testr command. pbr requires a git
|
||||
# version or an sdist to work. so make the test directory a git repo
|
||||
# too.
|
||||
subprocess.call(['git', 'init'], stderr=DEVNULL)
|
||||
self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0)
|
||||
|
||||
def test_tempest_run_fails(self):
|
||||
# Git init is required for the pbr testr command. pbr requires a git
|
||||
# version or an sdist to work. so make the test directory a git repo
|
||||
# too.
|
||||
subprocess.call(['git', 'init'], stderr=DEVNULL)
|
||||
self.assertRunExit(['tempest', 'run'], 1)
|
Loading…
Reference in New Issue
Block a user