From 8a4396e3d3b48447a1ea1b9c20810e1cc3a6c357 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Fri, 15 Sep 2017 12:18:10 +0530 Subject: [PATCH] Switch Tempest CLI commands from testrepository to stestr This commit switches the Tempest CLI commands to internally use stestr instead of testrepository. At this point in time the testrepository project is effectively unmaintained and stestr was a fork started to have an actively maintained test runner. It also focuses on being a dedicated python test runner, instead of an abstract test runner interface for any tests that emit subunit. Besides the bug fixes and other improvements included with stestr, this switch provides a number of advantages for tempest. Primarily stestr has a real python API for invoking the test runner directly from python. This means we can simplify the wrapper code to simply call a function instead of building out a set of CLI arguments and passing that to the CLI processor. Co-Authored-By: Matthew Treinish Depends-On: Ic1fa3a98b6bcd151c489b078028687892655a19b Depends-On: I3855aad5ce129ec8ccb87c05f7aa709b74070efe Depends-On: https://review.openstack.org/529490/ Change-Id: I6f5fa7796c576b71c4a0dde66896974a8039a848 --- .stestr.conf | 1 - .testr.conf | 9 -- .../switch-to-stestr-8c9f834b3f5a55d8.yaml | 13 ++ requirements.txt | 3 +- tempest/cmd/init.py | 35 ++--- tempest/cmd/run.py | 148 ++++-------------- tempest/tests/cmd/test_run.py | 65 +------- tempest/tests/cmd/test_tempest_init.py | 16 +- tempest/tests/files/testr-conf | 4 +- 9 files changed, 79 insertions(+), 215 deletions(-) delete mode 100644 .testr.conf create mode 100644 releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml diff --git a/.stestr.conf b/.stestr.conf index e3201c13c0..818c7436aa 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,4 +1,3 @@ [DEFAULT] test_path=./tempest/test_discover group_regex=([^\.]*\.)* - diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 95a4fb4d23..0000000000 --- a/.testr.conf +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \ - OS_TEST_LOCK_PATH=${OS_TEST_LOCK_PATH:-${TMPDIR:-'/tmp'}} \ - ${PYTHON:-python} -m subunit.run discover -t ${OS_TOP_LEVEL:-./} ${OS_TEST_PATH:-./tempest/test_discover} $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list -group_regex=([^\.]*\.)* diff --git a/releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml b/releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml new file mode 100644 index 0000000000..9e2f1bafb3 --- /dev/null +++ b/releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml @@ -0,0 +1,13 @@ +--- +features: +- The Tempest CLI commands have switched from calling testrepository internally + to use stestr instead. This means that all of the features and bug fixes from + moving to stestr are available to the tempest commands. + +upgrade: +- Tempest CLI commands will no long rely on anything from testr. This means any + data in existing testr internals that were being exposed are no longer + present. For example things like the .testr directories will be silently + ignored. There is a potential incompatibility for existing users who are + relying on test results being stored by testr. Anything relying on previous + testr behavior will need to be updated to handle stestr. diff --git a/requirements.txt b/requirements.txt index c02cd05ef5..76db574d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ jsonschema<3.0.0,>=2.6.0 # MIT testtools>=2.2.0 # MIT paramiko>=2.0.0 # LGPLv2.1+ netaddr>=0.7.18 # BSD -testrepository>=0.0.18 # Apache-2.0/BSD oslo.concurrency>=3.25.0 # Apache-2.0 oslo.config>=5.1.0 # Apache-2.0 oslo.log>=3.36.0 # Apache-2.0 +stestr>=1.0.0 # Apache-2.0 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 oslo.utils>=3.33.0 # Apache-2.0 six>=1.10.0 # MIT @@ -19,7 +19,6 @@ PyYAML>=3.10 # MIT python-subunit>=1.0.0 # Apache-2.0/BSD stevedore>=1.20.0 # Apache-2.0 PrettyTable<0.8,>=0.7.1 # BSD -os-testr>=1.0.0 # Apache-2.0 urllib3>=1.21.1 # MIT debtcollector>=1.2.0 # Apache-2.0 unittest2>=1.1.0 # BSD diff --git a/tempest/cmd/init.py b/tempest/cmd/init.py index 7634d9e1a1..9a85d8960c 100644 --- a/tempest/cmd/init.py +++ b/tempest/cmd/init.py @@ -20,19 +20,15 @@ from cliff import command from oslo_config import generator from oslo_log import log as logging from six import moves -from testrepository import commands +from stestr import commands from tempest.cmd import workspace LOG = logging.getLogger(__name__) -TESTR_CONF = """[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \\ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \\ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \\ - ${PYTHON:-python} -m subunit.run discover -t %s %s $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list +STESTR_CONF = """[DEFAULT] +test_path=%s +top_dir=%s group_regex=([^\.]*\.)* """ @@ -84,13 +80,13 @@ class TempestInit(command.Command): "is ~/.tempest/workspace.yaml") return parser - def generate_testr_conf(self, local_path): - testr_conf_path = os.path.join(local_path, '.testr.conf') + def generate_stestr_conf(self, local_path): + stestr_conf_path = os.path.join(local_path, '.stestr.conf') top_level_path = os.path.dirname(os.path.dirname(__file__)) discover_path = os.path.join(top_level_path, 'test_discover') - testr_conf = TESTR_CONF % (top_level_path, discover_path) - with open(testr_conf_path, 'w+') as testr_conf_file: - testr_conf_file.write(testr_conf) + stestr_conf = STESTR_CONF % (discover_path, top_level_path) + with open(stestr_conf_path, 'w+') as stestr_conf_file: + stestr_conf_file.write(stestr_conf) def get_configparser(self, conf_path): config_parse = moves.configparser.ConfigParser() @@ -148,7 +144,7 @@ class TempestInit(command.Command): etc_dir = os.path.join(local_dir, 'etc') config_path = os.path.join(etc_dir, 'tempest.conf') log_dir = os.path.join(local_dir, 'logs') - testr_dir = os.path.join(local_dir, '.testrepository') + stestr_dir = os.path.join(local_dir, '.stestr') # Create lock dir if not os.path.isdir(lock_dir): LOG.debug('Creating lock dir: %s', lock_dir) @@ -163,12 +159,11 @@ class TempestInit(command.Command): self.generate_sample_config(local_dir) # Update local confs to reflect local paths self.update_local_conf(config_path, lock_dir, log_dir) - # Generate a testr conf file - self.generate_testr_conf(local_dir) - # setup local testr working dir - if not os.path.isdir(testr_dir): - commands.run_argv(['testr', 'init', '-d', local_dir], sys.stdin, - sys.stdout, sys.stderr) + # Generate a stestr conf file + self.generate_stestr_conf(local_dir) + # setup local stestr working dir + if not os.path.isdir(stestr_dir): + commands.init_command(repo_url=local_dir) def take_action(self, parsed_args): workspace_manager = workspace.WorkspaceManager( diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py index 6435717b70..986602b884 100644 --- a/tempest/cmd/run.py +++ b/tempest/cmd/run.py @@ -19,9 +19,9 @@ Test Selection ============== Tempest run has several options: -* ``--regex, -r``: This is a selection regex like what testr uses. It will run - any tests that match on re.match() with the regex -* ``--smoke, -s``: Run all the tests tagged as smoke + * **--regex/-r**: This is a selection regex like what stestr uses. It will run + any tests that match on re.match() with the regex + * **--smoke/-s**: Run all the tests tagged as smoke There are also the ``--blacklist-file`` and ``--whitelist-file`` options that let you pass a filepath to tempest run with the file format being a line @@ -74,9 +74,9 @@ Running from Anywhere --------------------- Tempest run provides you with an option to execute tempest from anywhere on your system. You are required to provide a config file in this case with the -``--config-file`` option. When run tempest will create a .testrepository -directory and a .testr.conf file in your current working directory. This way -you can use testr commands directly to inspect the state of the previous run. +``--config-file`` option. When run tempest will create a .stestr +directory and a .stestr.conf file in your current working directory. This way +you can use stestr commands directly to inspect the state of the previous run. Test Output =========== @@ -94,18 +94,13 @@ as a single run you can leverage the ``--combine`` option which will append the current run's results with the previous runs. """ -import io import os import sys -import tempfile -import threading from cliff import command -from os_testr import regex_builder -from os_testr import subunit_trace from oslo_serialization import jsonutils as json import six -from testrepository.commands import run_argv +from stestr import commands from tempest import clients from tempest.cmd import cleanup_service @@ -124,35 +119,27 @@ class TempestRun(command.Command): def _set_env(self, config_file=None): if config_file: CONF.set_config_path(os.path.abspath(config_file)) - # NOTE(mtreinish): This is needed so that testr doesn't gobble up any + # NOTE(mtreinish): This is needed so that stestr doesn't gobble up any # stacktraces on failure. if 'TESTR_PDB' in os.environ: return else: os.environ["TESTR_PDB"] = "" - # NOTE(dims): most of our .testr.conf try to test for PYTHON + # NOTE(dims): most of our .stestr.conf try to test for PYTHON # environment variable and fall back to "python", under python3 # if it does not exist. we should set it to the python3 executable # to deal with this situation better for now. if six.PY3 and 'PYTHON' not in os.environ: os.environ['PYTHON'] = sys.executable - def _create_testrepository(self): - if not os.path.isdir('.testrepository'): - returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout, - sys.stderr) - if returncode: - sys.exit(returncode) - - def _create_testr_conf(self): + def _create_stestr_conf(self): top_level_path = os.path.dirname(os.path.dirname(__file__)) discover_path = os.path.join(top_level_path, 'test_discover') - file_contents = init.TESTR_CONF % (top_level_path, discover_path) - with open('.testr.conf', 'w+') as testr_conf_file: - testr_conf_file.write(file_contents) + file_contents = init.STESTR_CONF % (discover_path, top_level_path) + with open('.stestr.conf', 'w+') as stestr_conf_file: + stestr_conf_file.write(file_contents) def take_action(self, parsed_args): - returncode = 0 if parsed_args.config_file: self._set_env(parsed_args.config_file) else: @@ -169,52 +156,32 @@ class TempestRun(command.Command): "register the workspace." % (parsed_args.workspace, workspace_mgr.path)) os.chdir(path) - # NOTE(mtreinish): tempest init should create a .testrepository dir - # but since workspaces can be imported let's sanity check and - # ensure that one is created - self._create_testrepository() - # Local execution mode - elif os.path.isfile('.testr.conf'): - # If you're running in local execution mode and there is not a - # testrepository dir create one - self._create_testrepository() + if not os.path.isfile('.stestr.conf'): + self._create_stestr_conf() # local execution with config file mode elif parsed_args.config_file: - self._create_testr_conf() - self._create_testrepository() - else: - print("No .testr.conf file was found for local execution") + self._create_stestr_conf() + self._create_stestrepository() + elif not os.path.isfile('.stestr.conf'): + print("No .stestr.conf file was found for local execution") sys.exit(2) if parsed_args.state: self._init_state() else: pass - if parsed_args.combine: - temp_stream = tempfile.NamedTemporaryFile() - return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin, - temp_stream, sys.stderr) + if not (parsed_args.config_file or parsed_args.workspace): + regex = self._build_regex(parsed_args) + serial = not parsed_args.parallel + return_code = commands.run_command( + filters=regex, subunit_out=parsed_args.subunit, + serial=serial, concurrency=parsed_args.concurrency, + blacklist_file=parsed_args.blacklist_file, + whitelist_file=parsed_args.whitelist_file, + load_list=parsed_args.load_list, combine=parsed_args.combine) if return_code > 0: sys.exit(return_code) - - regex = self._build_regex(parsed_args) - if parsed_args.list_tests: - argv = ['tempest', 'list-tests', regex] - returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr) - else: - options = self._build_options(parsed_args) - returncode = self._run(regex, options) - if returncode > 0: - sys.exit(returncode) - - if parsed_args.combine: - return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin, - temp_stream, sys.stderr) - if return_code > 0: - sys.exit(return_code) - returncode = run_argv(['tempest', 'load', temp_stream.name], - sys.stdin, sys.stdout, sys.stderr) - sys.exit(returncode) + return return_code def get_description(self): return 'Run tempest' @@ -262,7 +229,7 @@ class TempestRun(command.Command): regex.add_argument('--smoke', '-s', action='store_true', help="Run the smoke tests only") regex.add_argument('--regex', '-r', default='', - help='A normal testr selection regex used to ' + help='A normal stestr selection regex used to ' 'specify a subset of tests to run') list_selector = parser.add_mutually_exclusive_group() list_selector.add_argument('--whitelist-file', '--whitelist_file', @@ -305,62 +272,15 @@ class TempestRun(command.Command): parser.add_argument("--combine", action='store_true', help='Combine the output of this run with the ' "previous run's as a combined stream in the " - "testr repository after it finish") + "stestr repository after it finish") parser.set_defaults(parallel=True) return parser def _build_regex(self, parsed_args): - regex = '' + regex = None if parsed_args.smoke: - regex = 'smoke' + regex = ['smoke'] elif parsed_args.regex: - regex = parsed_args.regex - if parsed_args.whitelist_file or parsed_args.blacklist_file: - regex = regex_builder.construct_regex(parsed_args.blacklist_file, - parsed_args.whitelist_file, - regex, False) + regex = parsed_args.regex.split() return regex - - def _build_options(self, parsed_args): - options = [] - if parsed_args.subunit: - options.append("--subunit") - if parsed_args.parallel: - options.append("--parallel") - if parsed_args.concurrency: - options.append("--concurrency=%s" % parsed_args.concurrency) - if parsed_args.load_list: - options.append("--load-list=%s" % parsed_args.load_list) - return options - - def _run(self, regex, options): - returncode = 0 - argv = ['tempest', 'run', regex] + options - if '--subunit' in options: - returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr) - else: - argv.append('--subunit') - stdin = io.StringIO() - stdout_r, stdout_w = os.pipe() - subunit_w = os.fdopen(stdout_w, 'wt') - subunit_r = os.fdopen(stdout_r) - returncodes = {} - - def run_argv_thread(): - returncodes['testr'] = run_argv(argv, stdin, subunit_w, - sys.stderr) - subunit_w.close() - - run_thread = threading.Thread(target=run_argv_thread) - run_thread.start() - returncodes['subunit-trace'] = subunit_trace.trace( - subunit_r, sys.stdout, post_fails=True, print_failures=True) - run_thread.join() - subunit_r.close() - # python version of pipefail - if returncodes['testr']: - returncode = returncodes['testr'] - elif returncodes['subunit-trace']: - returncode = returncodes['subunit-trace'] - return returncode diff --git a/tempest/tests/cmd/test_run.py b/tempest/tests/cmd/test_run.py index 0485e140ac..b2fddc9b12 100644 --- a/tempest/tests/cmd/test_run.py +++ b/tempest/tests/cmd/test_run.py @@ -35,24 +35,13 @@ class TestTempestRun(base.TestCase): super(TestTempestRun, self).setUp() self.run_cmd = run.TempestRun(None, None) - def test_build_options(self): - args = mock.Mock(spec=argparse.Namespace) - setattr(args, "subunit", True) - setattr(args, "parallel", False) - setattr(args, "concurrency", 10) - setattr(args, "load_list", '') - options = self.run_cmd._build_options(args) - self.assertEqual(['--subunit', - '--concurrency=10'], - options) - def test__build_regex_default(self): args = mock.Mock(spec=argparse.Namespace) setattr(args, 'smoke', False) setattr(args, 'regex', '') setattr(args, 'whitelist_file', None) setattr(args, 'blacklist_file', None) - self.assertEqual('', self.run_cmd._build_regex(args)) + self.assertIsNone(None, self.run_cmd._build_regex(args)) def test__build_regex_smoke(self): args = mock.Mock(spec=argparse.Namespace) @@ -60,7 +49,7 @@ class TestTempestRun(base.TestCase): setattr(args, 'regex', '') setattr(args, 'whitelist_file', None) setattr(args, 'blacklist_file', None) - self.assertEqual('smoke', self.run_cmd._build_regex(args)) + self.assertEqual(['smoke'], self.run_cmd._build_regex(args)) def test__build_regex_regex(self): args = mock.Mock(spec=argparse.Namespace) @@ -68,37 +57,9 @@ class TestTempestRun(base.TestCase): setattr(args, "regex", 'i_am_a_fun_little_regex') setattr(args, 'whitelist_file', None) setattr(args, 'blacklist_file', None) - self.assertEqual('i_am_a_fun_little_regex', + self.assertEqual(['i_am_a_fun_little_regex'], self.run_cmd._build_regex(args)) - def test__build_whitelist_file(self): - args = mock.Mock(spec=argparse.Namespace) - setattr(args, 'smoke', False) - setattr(args, 'regex', None) - self.tests = tempfile.NamedTemporaryFile( - prefix='whitelist', delete=False) - self.tests.write(b"volume \n compute") - self.tests.close() - setattr(args, 'whitelist_file', self.tests.name) - setattr(args, 'blacklist_file', None) - self.assertEqual("volume|compute", - self.run_cmd._build_regex(args)) - os.unlink(self.tests.name) - - def test__build_blacklist_file(self): - args = mock.Mock(spec=argparse.Namespace) - setattr(args, 'smoke', False) - setattr(args, 'regex', None) - self.tests = tempfile.NamedTemporaryFile( - prefix='blacklist', delete=False) - self.tests.write(b"volume \n compute") - self.tests.close() - setattr(args, 'whitelist_file', None) - setattr(args, 'blacklist_file', self.tests.name) - self.assertEqual("^((?!compute|volume).)*$", - self.run_cmd._build_regex(args)) - os.unlink(self.tests.name) - class TestRunReturnCode(base.TestCase): def setUp(self): @@ -109,13 +70,13 @@ class TestRunReturnCode(base.TestCase): self.test_dir = os.path.join(self.directory, 'tests') os.mkdir(self.test_dir) # Setup Test files - self.testr_conf_file = os.path.join(self.directory, '.testr.conf') + self.stestr_conf_file = os.path.join(self.directory, '.stestr.conf') self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg') self.passing_file = os.path.join(self.test_dir, 'test_passing.py') self.failing_file = os.path.join(self.test_dir, 'test_failing.py') self.init_file = os.path.join(self.test_dir, '__init__.py') self.setup_py = os.path.join(self.directory, 'setup.py') - shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file) + shutil.copy('tempest/tests/files/testr-conf', self.stestr_conf_file) shutil.copy('tempest/tests/files/passing-tests', self.passing_file) shutil.copy('tempest/tests/files/failing-tests', self.failing_file) shutil.copy('setup.py', self.setup_py) @@ -134,25 +95,13 @@ class TestRunReturnCode(base.TestCase): self.assertEqual(p.returncode, expected, msg) def test_tempest_run_passes(self): - # Git init is required for the pbr testr command. pbr requires a git - # version or an sdist to work. so make the test directory a git repo - # too. - subprocess.call(['git', 'init'], stderr=DEVNULL) self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0) - def test_tempest_run_passes_with_testrepository(self): - # Git init is required for the pbr testr command. pbr requires a git - # version or an sdist to work. so make the test directory a git repo - # too. - subprocess.call(['git', 'init'], stderr=DEVNULL) - subprocess.call(['testr', 'init']) + def test_tempest_run_passes_with_stestr_repository(self): + subprocess.call(['stestr', 'init']) self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0) def test_tempest_run_fails(self): - # Git init is required for the pbr testr command. pbr requires a git - # version or an sdist to work. so make the test directory a git repo - # too. - subprocess.call(['git', 'init'], stderr=DEVNULL) self.assertRunExit(['tempest', 'run'], 1) diff --git a/tempest/tests/cmd/test_tempest_init.py b/tempest/tests/cmd/test_tempest_init.py index 79510be763..5f39ac99a2 100644 --- a/tempest/tests/cmd/test_tempest_init.py +++ b/tempest/tests/cmd/test_tempest_init.py @@ -27,16 +27,16 @@ class TestTempestInit(base.TestCase): conf_dir = self.useFixture(fixtures.TempDir()) init_cmd = init.TempestInit(None, None) - init_cmd.generate_testr_conf(conf_dir.path) + init_cmd.generate_stestr_conf(conf_dir.path) # Generate expected file contents top_level_path = os.path.dirname(os.path.dirname(init.__file__)) discover_path = os.path.join(top_level_path, 'test_discover') - testr_conf_file = init.TESTR_CONF % (top_level_path, discover_path) + stestr_conf_file = init.STESTR_CONF % (discover_path, top_level_path) - conf_path = conf_dir.join('.testr.conf') + conf_path = conf_dir.join('.stestr.conf') with open(conf_path, 'r') as conf_file: - self.assertEqual(conf_file.read(), testr_conf_file) + self.assertEqual(conf_file.read(), stestr_conf_file) def test_generate_sample_config(self): local_dir = self.useFixture(fixtures.TempDir()) @@ -125,18 +125,18 @@ class TestTempestInit(base.TestCase): lock_path = os.path.join(fake_local_dir.path, 'tempest_lock') etc_dir = os.path.join(fake_local_dir.path, 'etc') log_dir = os.path.join(fake_local_dir.path, 'logs') - testr_dir = os.path.join(fake_local_dir.path, '.testrepository') + stestr_dir = os.path.join(fake_local_dir.path, '.stestr') self.assertTrue(os.path.isdir(lock_path)) self.assertTrue(os.path.isdir(etc_dir)) self.assertTrue(os.path.isdir(log_dir)) - self.assertTrue(os.path.isdir(testr_dir)) + self.assertTrue(os.path.isdir(stestr_dir)) # Assert file creation fake_file_moved = os.path.join(etc_dir, 'conf_file.conf') local_conf_file = os.path.join(etc_dir, 'tempest.conf') - local_testr_conf = os.path.join(fake_local_dir.path, '.testr.conf') + local_stestr_conf = os.path.join(fake_local_dir.path, '.stestr.conf') self.assertTrue(os.path.isfile(fake_file_moved)) self.assertTrue(os.path.isfile(local_conf_file)) - self.assertTrue(os.path.isfile(local_testr_conf)) + self.assertTrue(os.path.isfile(local_stestr_conf)) def test_take_action_fails(self): class ParsedArgs(object): diff --git a/tempest/tests/files/testr-conf b/tempest/tests/files/testr-conf index d5ad083225..63b3c44c2a 100644 --- a/tempest/tests/files/testr-conf +++ b/tempest/tests/files/testr-conf @@ -1,5 +1,3 @@ [DEFAULT] -test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list +test_path=./tests group_regex=([^\.]*\.)*