openstack/tempest
Code Issues Proposed changes

Merge "Switch Tempest CLI commands from testrepository to stestr"

Browse Source
This commit is contained in:
Zuul 2018-02-21 23:01:58 +00:00 committed by Gerrit Code Review
commit 24a753dcfb
9 changed files with 79 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.stestr.conf
View File

@ -1,4 +1,3 @@
[DEFAULT]
test_path=./tempest/test_discover
group_regex=([^\.]*\.)*

9
.testr.conf
View File

@ -1,9 +0,0 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \
OS_TEST_LOCK_PATH=${OS_TEST_LOCK_PATH:-${TMPDIR:-'/tmp'}} \
${PYTHON:-python} -m subunit.run discover -t ${OS_TOP_LEVEL:-./} ${OS_TEST_PATH:-./tempest/test_discover} $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
group_regex=([^\.]*\.)*

13
releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml Normal file
View File

@ -0,0 +1,13 @@
---
features:
- The Tempest CLI commands have switched from calling testrepository internally
to use stestr instead. This means that all of the features and bug fixes from
moving to stestr are available to the tempest commands.
upgrade:
- Tempest CLI commands will no long rely on anything from testr. This means any
data in existing testr internals that were being exposed are no longer
present. For example things like the .testr directories will be silently
ignored. There is a potential incompatibility for existing users who are
relying on test results being stored by testr. Anything relying on previous
testr behavior will need to be updated to handle stestr.

3
requirements.txt
View File

@ -7,10 +7,10 @@ jsonschema<3.0.0,>=2.6.0 # MIT
testtools>=2.2.0 # MIT
paramiko>=2.0.0 # LGPLv2.1+
netaddr>=0.7.18 # BSD
testrepository>=0.0.18 # Apache-2.0/BSD
oslo.concurrency>=3.25.0 # Apache-2.0
oslo.config>=5.1.0 # Apache-2.0
oslo.log>=3.36.0 # Apache-2.0
stestr>=1.0.0 # Apache-2.0
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
oslo.utils>=3.33.0 # Apache-2.0
six>=1.10.0 # MIT
@ -19,7 +19,6 @@ PyYAML>=3.10 # MIT
python-subunit>=1.0.0 # Apache-2.0/BSD
stevedore>=1.20.0 # Apache-2.0
PrettyTable<0.8,>=0.7.1 # BSD
os-testr>=1.0.0 # Apache-2.0
urllib3>=1.21.1 # MIT
debtcollector>=1.2.0 # Apache-2.0
unittest2>=1.1.0 # BSD

35
tempest/cmd/init.py
View File

@ -20,19 +20,15 @@ from cliff import command
from oslo_config import generator
from oslo_log import log as logging
from six import moves
from testrepository import commands
from stestr import commands
from tempest.cmd import workspace
LOG = logging.getLogger(__name__)
TESTR_CONF = """[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \\
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \\
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \\
${PYTHON:-python} -m subunit.run discover -t %s %s $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
STESTR_CONF = """[DEFAULT]
test_path=%s
top_dir=%s
group_regex=([^\.]*\.)*
"""
@ -84,13 +80,13 @@ class TempestInit(command.Command):
"is ~/.tempest/workspace.yaml")
return parser
def generate_testr_conf(self, local_path):
testr_conf_path = os.path.join(local_path, '.testr.conf')
def generate_stestr_conf(self, local_path):
stestr_conf_path = os.path.join(local_path, '.stestr.conf')
top_level_path = os.path.dirname(os.path.dirname(__file__))
discover_path = os.path.join(top_level_path, 'test_discover')
testr_conf = TESTR_CONF % (top_level_path, discover_path)
with open(testr_conf_path, 'w+') as testr_conf_file:
testr_conf_file.write(testr_conf)
stestr_conf = STESTR_CONF % (discover_path, top_level_path)
with open(stestr_conf_path, 'w+') as stestr_conf_file:
stestr_conf_file.write(stestr_conf)
def get_configparser(self, conf_path):
config_parse = moves.configparser.ConfigParser()
@ -148,7 +144,7 @@ class TempestInit(command.Command):
etc_dir = os.path.join(local_dir, 'etc')
config_path = os.path.join(etc_dir, 'tempest.conf')
log_dir = os.path.join(local_dir, 'logs')
testr_dir = os.path.join(local_dir, '.testrepository')
stestr_dir = os.path.join(local_dir, '.stestr')
# Create lock dir
if not os.path.isdir(lock_dir):
LOG.debug('Creating lock dir: %s', lock_dir)
@ -163,12 +159,11 @@ class TempestInit(command.Command):
self.generate_sample_config(local_dir)
# Update local confs to reflect local paths
self.update_local_conf(config_path, lock_dir, log_dir)
# Generate a testr conf file
self.generate_testr_conf(local_dir)
# setup local testr working dir
if not os.path.isdir(testr_dir):
commands.run_argv(['testr', 'init', '-d', local_dir], sys.stdin,
sys.stdout, sys.stderr)
# Generate a stestr conf file
self.generate_stestr_conf(local_dir)
# setup local stestr working dir
if not os.path.isdir(stestr_dir):
commands.init_command(repo_url=local_dir)
def take_action(self, parsed_args):
workspace_manager = workspace.WorkspaceManager(

148
tempest/cmd/run.py
View File

@ -19,9 +19,9 @@ Test Selection
==============
Tempest run has several options:
* ``--regex, -r``: This is a selection regex like what testr uses. It will run
any tests that match on re.match() with the regex
* ``--smoke, -s``: Run all the tests tagged as smoke
* **--regex/-r**: This is a selection regex like what stestr uses. It will run
any tests that match on re.match() with the regex
* **--smoke/-s**: Run all the tests tagged as smoke
There are also the ``--blacklist-file`` and ``--whitelist-file`` options that
let you pass a filepath to tempest run with the file format being a line
@ -74,9 +74,9 @@ Running from Anywhere
---------------------
Tempest run provides you with an option to execute tempest from anywhere on
your system. You are required to provide a config file in this case with the
``--config-file`` option. When run tempest will create a .testrepository
directory and a .testr.conf file in your current working directory. This way
you can use testr commands directly to inspect the state of the previous run.
``--config-file`` option. When run tempest will create a .stestr
directory and a .stestr.conf file in your current working directory. This way
you can use stestr commands directly to inspect the state of the previous run.
Test Output
===========
@ -94,18 +94,13 @@ as a single run you can leverage the ``--combine`` option which will append
the current run's results with the previous runs.
"""
import io
import os
import sys
import tempfile
import threading
from cliff import command
from os_testr import regex_builder
from os_testr import subunit_trace
from oslo_serialization import jsonutils as json
import six
from testrepository.commands import run_argv
from stestr import commands
from tempest import clients
from tempest.cmd import cleanup_service
@ -124,35 +119,27 @@ class TempestRun(command.Command):
def _set_env(self, config_file=None):
if config_file:
CONF.set_config_path(os.path.abspath(config_file))
# NOTE(mtreinish): This is needed so that testr doesn't gobble up any
# NOTE(mtreinish): This is needed so that stestr doesn't gobble up any
# stacktraces on failure.
if 'TESTR_PDB' in os.environ:
return
else:
os.environ["TESTR_PDB"] = ""
# NOTE(dims): most of our .testr.conf try to test for PYTHON
# NOTE(dims): most of our .stestr.conf try to test for PYTHON
# environment variable and fall back to "python", under python3
# if it does not exist. we should set it to the python3 executable
# to deal with this situation better for now.
if six.PY3 and 'PYTHON' not in os.environ:
os.environ['PYTHON'] = sys.executable
def _create_testrepository(self):
if not os.path.isdir('.testrepository'):
returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout,
sys.stderr)
if returncode:
sys.exit(returncode)
def _create_testr_conf(self):
def _create_stestr_conf(self):
top_level_path = os.path.dirname(os.path.dirname(__file__))
discover_path = os.path.join(top_level_path, 'test_discover')
file_contents = init.TESTR_CONF % (top_level_path, discover_path)
with open('.testr.conf', 'w+') as testr_conf_file:
testr_conf_file.write(file_contents)
file_contents = init.STESTR_CONF % (discover_path, top_level_path)
with open('.stestr.conf', 'w+') as stestr_conf_file:
stestr_conf_file.write(file_contents)
def take_action(self, parsed_args):
returncode = 0
if parsed_args.config_file:
self._set_env(parsed_args.config_file)
else:
@ -169,52 +156,32 @@ class TempestRun(command.Command):
"register the workspace." %
(parsed_args.workspace, workspace_mgr.path))
os.chdir(path)
# NOTE(mtreinish): tempest init should create a .testrepository dir
# but since workspaces can be imported let's sanity check and
# ensure that one is created
self._create_testrepository()
# Local execution mode
elif os.path.isfile('.testr.conf'):
# If you're running in local execution mode and there is not a
# testrepository dir create one
self._create_testrepository()
if not os.path.isfile('.stestr.conf'):
self._create_stestr_conf()
# local execution with config file mode
elif parsed_args.config_file:
self._create_testr_conf()
self._create_testrepository()
else:
print("No .testr.conf file was found for local execution")
self._create_stestr_conf()
self._create_stestrepository()
elif not os.path.isfile('.stestr.conf'):
print("No .stestr.conf file was found for local execution")
sys.exit(2)
if parsed_args.state:
self._init_state()
else:
pass
if parsed_args.combine:
temp_stream = tempfile.NamedTemporaryFile()
return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin,
temp_stream, sys.stderr)
if not (parsed_args.config_file or parsed_args.workspace):
regex = self._build_regex(parsed_args)
serial = not parsed_args.parallel
return_code = commands.run_command(
filters=regex, subunit_out=parsed_args.subunit,
serial=serial, concurrency=parsed_args.concurrency,
blacklist_file=parsed_args.blacklist_file,
whitelist_file=parsed_args.whitelist_file,
load_list=parsed_args.load_list, combine=parsed_args.combine)
if return_code > 0:
sys.exit(return_code)
regex = self._build_regex(parsed_args)
if parsed_args.list_tests:
argv = ['tempest', 'list-tests', regex]
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
else:
options = self._build_options(parsed_args)
returncode = self._run(regex, options)
if returncode > 0:
sys.exit(returncode)
if parsed_args.combine:
return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin,
temp_stream, sys.stderr)
if return_code > 0:
sys.exit(return_code)
returncode = run_argv(['tempest', 'load', temp_stream.name],
sys.stdin, sys.stdout, sys.stderr)
sys.exit(returncode)
return return_code
def get_description(self):
return 'Run tempest'
@ -262,7 +229,7 @@ class TempestRun(command.Command):
regex.add_argument('--smoke', '-s', action='store_true',
help="Run the smoke tests only")
regex.add_argument('--regex', '-r', default='',
help='A normal testr selection regex used to '
help='A normal stestr selection regex used to '
'specify a subset of tests to run')
list_selector = parser.add_mutually_exclusive_group()
list_selector.add_argument('--whitelist-file', '--whitelist_file',
@ -305,62 +272,15 @@ class TempestRun(command.Command):
parser.add_argument("--combine", action='store_true',
help='Combine the output of this run with the '
"previous run's as a combined stream in the "
"testr repository after it finish")
"stestr repository after it finish")
parser.set_defaults(parallel=True)
return parser
def _build_regex(self, parsed_args):
regex = ''
regex = None
if parsed_args.smoke:
regex = 'smoke'
regex = ['smoke']
elif parsed_args.regex:
regex = parsed_args.regex
if parsed_args.whitelist_file or parsed_args.blacklist_file:
regex = regex_builder.construct_regex(parsed_args.blacklist_file,
parsed_args.whitelist_file,
regex, False)
regex = parsed_args.regex.split()
return regex
def _build_options(self, parsed_args):
options = []
if parsed_args.subunit:
options.append("--subunit")
if parsed_args.parallel:
options.append("--parallel")
if parsed_args.concurrency:
options.append("--concurrency=%s" % parsed_args.concurrency)
if parsed_args.load_list:
options.append("--load-list=%s" % parsed_args.load_list)
return options
def _run(self, regex, options):
returncode = 0
argv = ['tempest', 'run', regex] + options
if '--subunit' in options:
returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
else:
argv.append('--subunit')
stdin = io.StringIO()
stdout_r, stdout_w = os.pipe()
subunit_w = os.fdopen(stdout_w, 'wt')
subunit_r = os.fdopen(stdout_r)
returncodes = {}
def run_argv_thread():
returncodes['testr'] = run_argv(argv, stdin, subunit_w,
sys.stderr)
subunit_w.close()
run_thread = threading.Thread(target=run_argv_thread)
run_thread.start()
returncodes['subunit-trace'] = subunit_trace.trace(
subunit_r, sys.stdout, post_fails=True, print_failures=True)
run_thread.join()
subunit_r.close()
# python version of pipefail
if returncodes['testr']:
returncode = returncodes['testr']
elif returncodes['subunit-trace']:
returncode = returncodes['subunit-trace']
return returncode

65
tempest/tests/cmd/test_run.py
View File

@ -35,24 +35,13 @@ class TestTempestRun(base.TestCase):
super(TestTempestRun, self).setUp()
self.run_cmd = run.TempestRun(None, None)
def test_build_options(self):
args = mock.Mock(spec=argparse.Namespace)
setattr(args, "subunit", True)
setattr(args, "parallel", False)
setattr(args, "concurrency", 10)
setattr(args, "load_list", '')
options = self.run_cmd._build_options(args)
self.assertEqual(['--subunit',
'--concurrency=10'],
options)
def test__build_regex_default(self):
args = mock.Mock(spec=argparse.Namespace)
setattr(args, 'smoke', False)
setattr(args, 'regex', '')
setattr(args, 'whitelist_file', None)
setattr(args, 'blacklist_file', None)
self.assertEqual('', self.run_cmd._build_regex(args))
self.assertIsNone(None, self.run_cmd._build_regex(args))
def test__build_regex_smoke(self):
args = mock.Mock(spec=argparse.Namespace)
@ -60,7 +49,7 @@ class TestTempestRun(base.TestCase):
setattr(args, 'regex', '')
setattr(args, 'whitelist_file', None)
setattr(args, 'blacklist_file', None)
self.assertEqual('smoke', self.run_cmd._build_regex(args))
self.assertEqual(['smoke'], self.run_cmd._build_regex(args))
def test__build_regex_regex(self):
args = mock.Mock(spec=argparse.Namespace)
@ -68,37 +57,9 @@ class TestTempestRun(base.TestCase):
setattr(args, "regex", 'i_am_a_fun_little_regex')
setattr(args, 'whitelist_file', None)
setattr(args, 'blacklist_file', None)
self.assertEqual('i_am_a_fun_little_regex',
self.assertEqual(['i_am_a_fun_little_regex'],
self.run_cmd._build_regex(args))
def test__build_whitelist_file(self):
args = mock.Mock(spec=argparse.Namespace)
setattr(args, 'smoke', False)
setattr(args, 'regex', None)
self.tests = tempfile.NamedTemporaryFile(
prefix='whitelist', delete=False)
self.tests.write(b"volume \n compute")
self.tests.close()
setattr(args, 'whitelist_file', self.tests.name)
setattr(args, 'blacklist_file', None)
self.assertEqual("volume|compute",
self.run_cmd._build_regex(args))
os.unlink(self.tests.name)
def test__build_blacklist_file(self):
args = mock.Mock(spec=argparse.Namespace)
setattr(args, 'smoke', False)
setattr(args, 'regex', None)
self.tests = tempfile.NamedTemporaryFile(
prefix='blacklist', delete=False)
self.tests.write(b"volume \n compute")
self.tests.close()
setattr(args, 'whitelist_file', None)
setattr(args, 'blacklist_file', self.tests.name)
self.assertEqual("^((?!compute|volume).)*$",
self.run_cmd._build_regex(args))
os.unlink(self.tests.name)
class TestRunReturnCode(base.TestCase):
def setUp(self):
@ -109,13 +70,13 @@ class TestRunReturnCode(base.TestCase):
self.test_dir = os.path.join(self.directory, 'tests')
os.mkdir(self.test_dir)
# Setup Test files
self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
self.stestr_conf_file = os.path.join(self.directory, '.stestr.conf')
self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
self.init_file = os.path.join(self.test_dir, '__init__.py')
self.setup_py = os.path.join(self.directory, 'setup.py')
shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
shutil.copy('tempest/tests/files/testr-conf', self.stestr_conf_file)
shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
shutil.copy('setup.py', self.setup_py)
@ -134,25 +95,13 @@ class TestRunReturnCode(base.TestCase):
self.assertEqual(p.returncode, expected, msg)
def test_tempest_run_passes(self):
# Git init is required for the pbr testr command. pbr requires a git
# version or an sdist to work. so make the test directory a git repo
# too.
subprocess.call(['git', 'init'], stderr=DEVNULL)
self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0)
def test_tempest_run_passes_with_testrepository(self):
# Git init is required for the pbr testr command. pbr requires a git
# version or an sdist to work. so make the test directory a git repo
# too.
subprocess.call(['git', 'init'], stderr=DEVNULL)
subprocess.call(['testr', 'init'])
def test_tempest_run_passes_with_stestr_repository(self):
subprocess.call(['stestr', 'init'])
self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0)
def test_tempest_run_fails(self):
# Git init is required for the pbr testr command. pbr requires a git
# version or an sdist to work. so make the test directory a git repo
# too.
subprocess.call(['git', 'init'], stderr=DEVNULL)
self.assertRunExit(['tempest', 'run'], 1)

16
tempest/tests/cmd/test_tempest_init.py
View File

@ -27,16 +27,16 @@ class TestTempestInit(base.TestCase):
conf_dir = self.useFixture(fixtures.TempDir())
init_cmd = init.TempestInit(None, None)
init_cmd.generate_testr_conf(conf_dir.path)
init_cmd.generate_stestr_conf(conf_dir.path)
# Generate expected file contents
top_level_path = os.path.dirname(os.path.dirname(init.__file__))
discover_path = os.path.join(top_level_path, 'test_discover')
testr_conf_file = init.TESTR_CONF % (top_level_path, discover_path)
stestr_conf_file = init.STESTR_CONF % (discover_path, top_level_path)
conf_path = conf_dir.join('.testr.conf')
conf_path = conf_dir.join('.stestr.conf')
with open(conf_path, 'r') as conf_file:
self.assertEqual(conf_file.read(), testr_conf_file)
self.assertEqual(conf_file.read(), stestr_conf_file)
def test_generate_sample_config(self):
local_dir = self.useFixture(fixtures.TempDir())
@ -125,18 +125,18 @@ class TestTempestInit(base.TestCase):
lock_path = os.path.join(fake_local_dir.path, 'tempest_lock')
etc_dir = os.path.join(fake_local_dir.path, 'etc')
log_dir = os.path.join(fake_local_dir.path, 'logs')
testr_dir = os.path.join(fake_local_dir.path, '.testrepository')
stestr_dir = os.path.join(fake_local_dir.path, '.stestr')
self.assertTrue(os.path.isdir(lock_path))
self.assertTrue(os.path.isdir(etc_dir))
self.assertTrue(os.path.isdir(log_dir))
self.assertTrue(os.path.isdir(testr_dir))
self.assertTrue(os.path.isdir(stestr_dir))
# Assert file creation
fake_file_moved = os.path.join(etc_dir, 'conf_file.conf')
local_conf_file = os.path.join(etc_dir, 'tempest.conf')
local_testr_conf = os.path.join(fake_local_dir.path, '.testr.conf')
local_stestr_conf = os.path.join(fake_local_dir.path, '.stestr.conf')
self.assertTrue(os.path.isfile(fake_file_moved))
self.assertTrue(os.path.isfile(local_conf_file))
self.assertTrue(os.path.isfile(local_testr_conf))
self.assertTrue(os.path.isfile(local_stestr_conf))
def test_take_action_fails(self):
class ParsedArgs(object):

4
tempest/tests/files/testr-conf
View File

@ -1,5 +1,3 @@
[DEFAULT]
test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
test_path=./tests
group_regex=([^\.]*\.)*