diff --git a/.stestr.conf b/.stestr.conf index e3201c13c0..818c7436aa 100644 --- a/.stestr.conf +++ b/.stestr.conf @@ -1,4 +1,3 @@ [DEFAULT] test_path=./tempest/test_discover group_regex=([^\.]*\.)* - diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index 95a4fb4d23..0000000000 --- a/.testr.conf +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \ - OS_TEST_LOCK_PATH=${OS_TEST_LOCK_PATH:-${TMPDIR:-'/tmp'}} \ - ${PYTHON:-python} -m subunit.run discover -t ${OS_TOP_LEVEL:-./} ${OS_TEST_PATH:-./tempest/test_discover} $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list -group_regex=([^\.]*\.)* diff --git a/releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml b/releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml new file mode 100644 index 0000000000..9e2f1bafb3 --- /dev/null +++ b/releasenotes/notes/switch-to-stestr-8c9f834b3f5a55d8.yaml @@ -0,0 +1,13 @@ +--- +features: +- The Tempest CLI commands have switched from calling testrepository internally + to use stestr instead. This means that all of the features and bug fixes from + moving to stestr are available to the tempest commands. + +upgrade: +- Tempest CLI commands will no long rely on anything from testr. This means any + data in existing testr internals that were being exposed are no longer + present. For example things like the .testr directories will be silently + ignored. There is a potential incompatibility for existing users who are + relying on test results being stored by testr. Anything relying on previous + testr behavior will need to be updated to handle stestr. diff --git a/requirements.txt b/requirements.txt index c02cd05ef5..76db574d2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,10 +7,10 @@ jsonschema<3.0.0,>=2.6.0 # MIT testtools>=2.2.0 # MIT paramiko>=2.0.0 # LGPLv2.1+ netaddr>=0.7.18 # BSD -testrepository>=0.0.18 # Apache-2.0/BSD oslo.concurrency>=3.25.0 # Apache-2.0 oslo.config>=5.1.0 # Apache-2.0 oslo.log>=3.36.0 # Apache-2.0 +stestr>=1.0.0 # Apache-2.0 oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 oslo.utils>=3.33.0 # Apache-2.0 six>=1.10.0 # MIT @@ -19,7 +19,6 @@ PyYAML>=3.10 # MIT python-subunit>=1.0.0 # Apache-2.0/BSD stevedore>=1.20.0 # Apache-2.0 PrettyTable<0.8,>=0.7.1 # BSD -os-testr>=1.0.0 # Apache-2.0 urllib3>=1.21.1 # MIT debtcollector>=1.2.0 # Apache-2.0 unittest2>=1.1.0 # BSD diff --git a/tempest/cmd/init.py b/tempest/cmd/init.py index 7634d9e1a1..9a85d8960c 100644 --- a/tempest/cmd/init.py +++ b/tempest/cmd/init.py @@ -20,19 +20,15 @@ from cliff import command from oslo_config import generator from oslo_log import log as logging from six import moves -from testrepository import commands +from stestr import commands from tempest.cmd import workspace LOG = logging.getLogger(__name__) -TESTR_CONF = """[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \\ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \\ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-500} \\ - ${PYTHON:-python} -m subunit.run discover -t %s %s $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list +STESTR_CONF = """[DEFAULT] +test_path=%s +top_dir=%s group_regex=([^\.]*\.)* """ @@ -84,13 +80,13 @@ class TempestInit(command.Command): "is ~/.tempest/workspace.yaml") return parser - def generate_testr_conf(self, local_path): - testr_conf_path = os.path.join(local_path, '.testr.conf') + def generate_stestr_conf(self, local_path): + stestr_conf_path = os.path.join(local_path, '.stestr.conf') top_level_path = os.path.dirname(os.path.dirname(__file__)) discover_path = os.path.join(top_level_path, 'test_discover') - testr_conf = TESTR_CONF % (top_level_path, discover_path) - with open(testr_conf_path, 'w+') as testr_conf_file: - testr_conf_file.write(testr_conf) + stestr_conf = STESTR_CONF % (discover_path, top_level_path) + with open(stestr_conf_path, 'w+') as stestr_conf_file: + stestr_conf_file.write(stestr_conf) def get_configparser(self, conf_path): config_parse = moves.configparser.ConfigParser() @@ -148,7 +144,7 @@ class TempestInit(command.Command): etc_dir = os.path.join(local_dir, 'etc') config_path = os.path.join(etc_dir, 'tempest.conf') log_dir = os.path.join(local_dir, 'logs') - testr_dir = os.path.join(local_dir, '.testrepository') + stestr_dir = os.path.join(local_dir, '.stestr') # Create lock dir if not os.path.isdir(lock_dir): LOG.debug('Creating lock dir: %s', lock_dir) @@ -163,12 +159,11 @@ class TempestInit(command.Command): self.generate_sample_config(local_dir) # Update local confs to reflect local paths self.update_local_conf(config_path, lock_dir, log_dir) - # Generate a testr conf file - self.generate_testr_conf(local_dir) - # setup local testr working dir - if not os.path.isdir(testr_dir): - commands.run_argv(['testr', 'init', '-d', local_dir], sys.stdin, - sys.stdout, sys.stderr) + # Generate a stestr conf file + self.generate_stestr_conf(local_dir) + # setup local stestr working dir + if not os.path.isdir(stestr_dir): + commands.init_command(repo_url=local_dir) def take_action(self, parsed_args): workspace_manager = workspace.WorkspaceManager( diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py index 6435717b70..986602b884 100644 --- a/tempest/cmd/run.py +++ b/tempest/cmd/run.py @@ -19,9 +19,9 @@ Test Selection ============== Tempest run has several options: -* ``--regex, -r``: This is a selection regex like what testr uses. It will run - any tests that match on re.match() with the regex -* ``--smoke, -s``: Run all the tests tagged as smoke + * **--regex/-r**: This is a selection regex like what stestr uses. It will run + any tests that match on re.match() with the regex + * **--smoke/-s**: Run all the tests tagged as smoke There are also the ``--blacklist-file`` and ``--whitelist-file`` options that let you pass a filepath to tempest run with the file format being a line @@ -74,9 +74,9 @@ Running from Anywhere --------------------- Tempest run provides you with an option to execute tempest from anywhere on your system. You are required to provide a config file in this case with the -``--config-file`` option. When run tempest will create a .testrepository -directory and a .testr.conf file in your current working directory. This way -you can use testr commands directly to inspect the state of the previous run. +``--config-file`` option. When run tempest will create a .stestr +directory and a .stestr.conf file in your current working directory. This way +you can use stestr commands directly to inspect the state of the previous run. Test Output =========== @@ -94,18 +94,13 @@ as a single run you can leverage the ``--combine`` option which will append the current run's results with the previous runs. """ -import io import os import sys -import tempfile -import threading from cliff import command -from os_testr import regex_builder -from os_testr import subunit_trace from oslo_serialization import jsonutils as json import six -from testrepository.commands import run_argv +from stestr import commands from tempest import clients from tempest.cmd import cleanup_service @@ -124,35 +119,27 @@ class TempestRun(command.Command): def _set_env(self, config_file=None): if config_file: CONF.set_config_path(os.path.abspath(config_file)) - # NOTE(mtreinish): This is needed so that testr doesn't gobble up any + # NOTE(mtreinish): This is needed so that stestr doesn't gobble up any # stacktraces on failure. if 'TESTR_PDB' in os.environ: return else: os.environ["TESTR_PDB"] = "" - # NOTE(dims): most of our .testr.conf try to test for PYTHON + # NOTE(dims): most of our .stestr.conf try to test for PYTHON # environment variable and fall back to "python", under python3 # if it does not exist. we should set it to the python3 executable # to deal with this situation better for now. if six.PY3 and 'PYTHON' not in os.environ: os.environ['PYTHON'] = sys.executable - def _create_testrepository(self): - if not os.path.isdir('.testrepository'): - returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout, - sys.stderr) - if returncode: - sys.exit(returncode) - - def _create_testr_conf(self): + def _create_stestr_conf(self): top_level_path = os.path.dirname(os.path.dirname(__file__)) discover_path = os.path.join(top_level_path, 'test_discover') - file_contents = init.TESTR_CONF % (top_level_path, discover_path) - with open('.testr.conf', 'w+') as testr_conf_file: - testr_conf_file.write(file_contents) + file_contents = init.STESTR_CONF % (discover_path, top_level_path) + with open('.stestr.conf', 'w+') as stestr_conf_file: + stestr_conf_file.write(file_contents) def take_action(self, parsed_args): - returncode = 0 if parsed_args.config_file: self._set_env(parsed_args.config_file) else: @@ -169,52 +156,32 @@ class TempestRun(command.Command): "register the workspace." % (parsed_args.workspace, workspace_mgr.path)) os.chdir(path) - # NOTE(mtreinish): tempest init should create a .testrepository dir - # but since workspaces can be imported let's sanity check and - # ensure that one is created - self._create_testrepository() - # Local execution mode - elif os.path.isfile('.testr.conf'): - # If you're running in local execution mode and there is not a - # testrepository dir create one - self._create_testrepository() + if not os.path.isfile('.stestr.conf'): + self._create_stestr_conf() # local execution with config file mode elif parsed_args.config_file: - self._create_testr_conf() - self._create_testrepository() - else: - print("No .testr.conf file was found for local execution") + self._create_stestr_conf() + self._create_stestrepository() + elif not os.path.isfile('.stestr.conf'): + print("No .stestr.conf file was found for local execution") sys.exit(2) if parsed_args.state: self._init_state() else: pass - if parsed_args.combine: - temp_stream = tempfile.NamedTemporaryFile() - return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin, - temp_stream, sys.stderr) + if not (parsed_args.config_file or parsed_args.workspace): + regex = self._build_regex(parsed_args) + serial = not parsed_args.parallel + return_code = commands.run_command( + filters=regex, subunit_out=parsed_args.subunit, + serial=serial, concurrency=parsed_args.concurrency, + blacklist_file=parsed_args.blacklist_file, + whitelist_file=parsed_args.whitelist_file, + load_list=parsed_args.load_list, combine=parsed_args.combine) if return_code > 0: sys.exit(return_code) - - regex = self._build_regex(parsed_args) - if parsed_args.list_tests: - argv = ['tempest', 'list-tests', regex] - returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr) - else: - options = self._build_options(parsed_args) - returncode = self._run(regex, options) - if returncode > 0: - sys.exit(returncode) - - if parsed_args.combine: - return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin, - temp_stream, sys.stderr) - if return_code > 0: - sys.exit(return_code) - returncode = run_argv(['tempest', 'load', temp_stream.name], - sys.stdin, sys.stdout, sys.stderr) - sys.exit(returncode) + return return_code def get_description(self): return 'Run tempest' @@ -262,7 +229,7 @@ class TempestRun(command.Command): regex.add_argument('--smoke', '-s', action='store_true', help="Run the smoke tests only") regex.add_argument('--regex', '-r', default='', - help='A normal testr selection regex used to ' + help='A normal stestr selection regex used to ' 'specify a subset of tests to run') list_selector = parser.add_mutually_exclusive_group() list_selector.add_argument('--whitelist-file', '--whitelist_file', @@ -305,62 +272,15 @@ class TempestRun(command.Command): parser.add_argument("--combine", action='store_true', help='Combine the output of this run with the ' "previous run's as a combined stream in the " - "testr repository after it finish") + "stestr repository after it finish") parser.set_defaults(parallel=True) return parser def _build_regex(self, parsed_args): - regex = '' + regex = None if parsed_args.smoke: - regex = 'smoke' + regex = ['smoke'] elif parsed_args.regex: - regex = parsed_args.regex - if parsed_args.whitelist_file or parsed_args.blacklist_file: - regex = regex_builder.construct_regex(parsed_args.blacklist_file, - parsed_args.whitelist_file, - regex, False) + regex = parsed_args.regex.split() return regex - - def _build_options(self, parsed_args): - options = [] - if parsed_args.subunit: - options.append("--subunit") - if parsed_args.parallel: - options.append("--parallel") - if parsed_args.concurrency: - options.append("--concurrency=%s" % parsed_args.concurrency) - if parsed_args.load_list: - options.append("--load-list=%s" % parsed_args.load_list) - return options - - def _run(self, regex, options): - returncode = 0 - argv = ['tempest', 'run', regex] + options - if '--subunit' in options: - returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr) - else: - argv.append('--subunit') - stdin = io.StringIO() - stdout_r, stdout_w = os.pipe() - subunit_w = os.fdopen(stdout_w, 'wt') - subunit_r = os.fdopen(stdout_r) - returncodes = {} - - def run_argv_thread(): - returncodes['testr'] = run_argv(argv, stdin, subunit_w, - sys.stderr) - subunit_w.close() - - run_thread = threading.Thread(target=run_argv_thread) - run_thread.start() - returncodes['subunit-trace'] = subunit_trace.trace( - subunit_r, sys.stdout, post_fails=True, print_failures=True) - run_thread.join() - subunit_r.close() - # python version of pipefail - if returncodes['testr']: - returncode = returncodes['testr'] - elif returncodes['subunit-trace']: - returncode = returncodes['subunit-trace'] - return returncode diff --git a/tempest/tests/cmd/test_run.py b/tempest/tests/cmd/test_run.py index 0485e140ac..b2fddc9b12 100644 --- a/tempest/tests/cmd/test_run.py +++ b/tempest/tests/cmd/test_run.py @@ -35,24 +35,13 @@ class TestTempestRun(base.TestCase): super(TestTempestRun, self).setUp() self.run_cmd = run.TempestRun(None, None) - def test_build_options(self): - args = mock.Mock(spec=argparse.Namespace) - setattr(args, "subunit", True) - setattr(args, "parallel", False) - setattr(args, "concurrency", 10) - setattr(args, "load_list", '') - options = self.run_cmd._build_options(args) - self.assertEqual(['--subunit', - '--concurrency=10'], - options) - def test__build_regex_default(self): args = mock.Mock(spec=argparse.Namespace) setattr(args, 'smoke', False) setattr(args, 'regex', '') setattr(args, 'whitelist_file', None) setattr(args, 'blacklist_file', None) - self.assertEqual('', self.run_cmd._build_regex(args)) + self.assertIsNone(None, self.run_cmd._build_regex(args)) def test__build_regex_smoke(self): args = mock.Mock(spec=argparse.Namespace) @@ -60,7 +49,7 @@ class TestTempestRun(base.TestCase): setattr(args, 'regex', '') setattr(args, 'whitelist_file', None) setattr(args, 'blacklist_file', None) - self.assertEqual('smoke', self.run_cmd._build_regex(args)) + self.assertEqual(['smoke'], self.run_cmd._build_regex(args)) def test__build_regex_regex(self): args = mock.Mock(spec=argparse.Namespace) @@ -68,37 +57,9 @@ class TestTempestRun(base.TestCase): setattr(args, "regex", 'i_am_a_fun_little_regex') setattr(args, 'whitelist_file', None) setattr(args, 'blacklist_file', None) - self.assertEqual('i_am_a_fun_little_regex', + self.assertEqual(['i_am_a_fun_little_regex'], self.run_cmd._build_regex(args)) - def test__build_whitelist_file(self): - args = mock.Mock(spec=argparse.Namespace) - setattr(args, 'smoke', False) - setattr(args, 'regex', None) - self.tests = tempfile.NamedTemporaryFile( - prefix='whitelist', delete=False) - self.tests.write(b"volume \n compute") - self.tests.close() - setattr(args, 'whitelist_file', self.tests.name) - setattr(args, 'blacklist_file', None) - self.assertEqual("volume|compute", - self.run_cmd._build_regex(args)) - os.unlink(self.tests.name) - - def test__build_blacklist_file(self): - args = mock.Mock(spec=argparse.Namespace) - setattr(args, 'smoke', False) - setattr(args, 'regex', None) - self.tests = tempfile.NamedTemporaryFile( - prefix='blacklist', delete=False) - self.tests.write(b"volume \n compute") - self.tests.close() - setattr(args, 'whitelist_file', None) - setattr(args, 'blacklist_file', self.tests.name) - self.assertEqual("^((?!compute|volume).)*$", - self.run_cmd._build_regex(args)) - os.unlink(self.tests.name) - class TestRunReturnCode(base.TestCase): def setUp(self): @@ -109,13 +70,13 @@ class TestRunReturnCode(base.TestCase): self.test_dir = os.path.join(self.directory, 'tests') os.mkdir(self.test_dir) # Setup Test files - self.testr_conf_file = os.path.join(self.directory, '.testr.conf') + self.stestr_conf_file = os.path.join(self.directory, '.stestr.conf') self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg') self.passing_file = os.path.join(self.test_dir, 'test_passing.py') self.failing_file = os.path.join(self.test_dir, 'test_failing.py') self.init_file = os.path.join(self.test_dir, '__init__.py') self.setup_py = os.path.join(self.directory, 'setup.py') - shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file) + shutil.copy('tempest/tests/files/testr-conf', self.stestr_conf_file) shutil.copy('tempest/tests/files/passing-tests', self.passing_file) shutil.copy('tempest/tests/files/failing-tests', self.failing_file) shutil.copy('setup.py', self.setup_py) @@ -134,25 +95,13 @@ class TestRunReturnCode(base.TestCase): self.assertEqual(p.returncode, expected, msg) def test_tempest_run_passes(self): - # Git init is required for the pbr testr command. pbr requires a git - # version or an sdist to work. so make the test directory a git repo - # too. - subprocess.call(['git', 'init'], stderr=DEVNULL) self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0) - def test_tempest_run_passes_with_testrepository(self): - # Git init is required for the pbr testr command. pbr requires a git - # version or an sdist to work. so make the test directory a git repo - # too. - subprocess.call(['git', 'init'], stderr=DEVNULL) - subprocess.call(['testr', 'init']) + def test_tempest_run_passes_with_stestr_repository(self): + subprocess.call(['stestr', 'init']) self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0) def test_tempest_run_fails(self): - # Git init is required for the pbr testr command. pbr requires a git - # version or an sdist to work. so make the test directory a git repo - # too. - subprocess.call(['git', 'init'], stderr=DEVNULL) self.assertRunExit(['tempest', 'run'], 1) diff --git a/tempest/tests/cmd/test_tempest_init.py b/tempest/tests/cmd/test_tempest_init.py index 79510be763..5f39ac99a2 100644 --- a/tempest/tests/cmd/test_tempest_init.py +++ b/tempest/tests/cmd/test_tempest_init.py @@ -27,16 +27,16 @@ class TestTempestInit(base.TestCase): conf_dir = self.useFixture(fixtures.TempDir()) init_cmd = init.TempestInit(None, None) - init_cmd.generate_testr_conf(conf_dir.path) + init_cmd.generate_stestr_conf(conf_dir.path) # Generate expected file contents top_level_path = os.path.dirname(os.path.dirname(init.__file__)) discover_path = os.path.join(top_level_path, 'test_discover') - testr_conf_file = init.TESTR_CONF % (top_level_path, discover_path) + stestr_conf_file = init.STESTR_CONF % (discover_path, top_level_path) - conf_path = conf_dir.join('.testr.conf') + conf_path = conf_dir.join('.stestr.conf') with open(conf_path, 'r') as conf_file: - self.assertEqual(conf_file.read(), testr_conf_file) + self.assertEqual(conf_file.read(), stestr_conf_file) def test_generate_sample_config(self): local_dir = self.useFixture(fixtures.TempDir()) @@ -125,18 +125,18 @@ class TestTempestInit(base.TestCase): lock_path = os.path.join(fake_local_dir.path, 'tempest_lock') etc_dir = os.path.join(fake_local_dir.path, 'etc') log_dir = os.path.join(fake_local_dir.path, 'logs') - testr_dir = os.path.join(fake_local_dir.path, '.testrepository') + stestr_dir = os.path.join(fake_local_dir.path, '.stestr') self.assertTrue(os.path.isdir(lock_path)) self.assertTrue(os.path.isdir(etc_dir)) self.assertTrue(os.path.isdir(log_dir)) - self.assertTrue(os.path.isdir(testr_dir)) + self.assertTrue(os.path.isdir(stestr_dir)) # Assert file creation fake_file_moved = os.path.join(etc_dir, 'conf_file.conf') local_conf_file = os.path.join(etc_dir, 'tempest.conf') - local_testr_conf = os.path.join(fake_local_dir.path, '.testr.conf') + local_stestr_conf = os.path.join(fake_local_dir.path, '.stestr.conf') self.assertTrue(os.path.isfile(fake_file_moved)) self.assertTrue(os.path.isfile(local_conf_file)) - self.assertTrue(os.path.isfile(local_testr_conf)) + self.assertTrue(os.path.isfile(local_stestr_conf)) def test_take_action_fails(self): class ParsedArgs(object): diff --git a/tempest/tests/files/testr-conf b/tempest/tests/files/testr-conf index d5ad083225..63b3c44c2a 100644 --- a/tempest/tests/files/testr-conf +++ b/tempest/tests/files/testr-conf @@ -1,5 +1,3 @@ [DEFAULT] -test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION -test_id_option=--load-list $IDFILE -test_list_option=--list +test_path=./tests group_regex=([^\.]*\.)*