|
|
|
@ -68,6 +68,11 @@ class TestTempestRun(base.TestCase):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestRunReturnCode(base.TestCase):
|
|
|
|
|
|
|
|
|
|
exclude_regex = '--exclude-regex'
|
|
|
|
|
exclude_list = '--exclude-list'
|
|
|
|
|
include_list = '--include-list'
|
|
|
|
|
|
|
|
|
|
def setUp(self):
|
|
|
|
|
super(TestRunReturnCode, self).setUp()
|
|
|
|
|
# Setup test dirs
|
|
|
|
@ -92,6 +97,14 @@ class TestRunReturnCode(base.TestCase):
|
|
|
|
|
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
|
|
|
|
|
os.chdir(self.directory)
|
|
|
|
|
|
|
|
|
|
def _get_test_list_file(self, content):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
test_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(test_file.close)
|
|
|
|
|
test_file.write(content.encode('utf-8'))
|
|
|
|
|
return path
|
|
|
|
|
|
|
|
|
|
def assertRunExit(self, cmd, expected):
|
|
|
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
|
|
|
|
|
stderr=subprocess.PIPE)
|
|
|
|
@ -115,19 +128,23 @@ class TestRunReturnCode(base.TestCase):
|
|
|
|
|
subprocess.call(['stestr', 'init'])
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--regex', 'failing'], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_blackregex_failing(self):
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--black-regex', 'failing'], 0)
|
|
|
|
|
def test_tempest_run_exclude_regex_failing(self):
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
self.exclude_regex, 'failing'], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_blackregex_failing_with_stestr_repository(self):
|
|
|
|
|
def test_tempest_run_exclude_regex_failing_with_stestr_repository(self):
|
|
|
|
|
subprocess.call(['stestr', 'init'])
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--black-regex', 'failing'], 0)
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
self.exclude_regex, 'failing'], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_blackregex_passing(self):
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--black-regex', 'passing'], 1)
|
|
|
|
|
def test_tempest_run_exclude_regex_passing(self):
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
self.exclude_regex, 'passing'], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_blackregex_passing_with_stestr_repository(self):
|
|
|
|
|
def test_tempest_run_exclude_regex_passing_with_stestr_repository(self):
|
|
|
|
|
subprocess.call(['stestr', 'init'])
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--black-regex', 'passing'], 1)
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
self.exclude_regex, 'passing'], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_fails(self):
|
|
|
|
|
self.assertRunExit(['tempest', 'run'], 1)
|
|
|
|
@ -149,47 +166,31 @@ class TestRunReturnCode(base.TestCase):
|
|
|
|
|
self.assertEqual(result, tests)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_worker_file(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
worker_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(worker_file.close)
|
|
|
|
|
worker_file.write(
|
|
|
|
|
'- worker:\n - passing\n concurrency: 3'.encode('utf-8'))
|
|
|
|
|
path = self._get_test_list_file(
|
|
|
|
|
'- worker:\n - passing\n concurrency: 3')
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--worker-file=%s' % path], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_whitelist(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
whitelist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(whitelist_file.close)
|
|
|
|
|
whitelist_file.write('passing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path], 0)
|
|
|
|
|
def test_tempest_run_with_include_list(self):
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.include_list, path)], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_whitelist_regex_include_pass_check_fail(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
whitelist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(whitelist_file.close)
|
|
|
|
|
whitelist_file.write('passing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path,
|
|
|
|
|
def test_tempest_run_with_include_regex_include_pass_check_fail(self):
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.include_list, path),
|
|
|
|
|
'--regex', 'fail'], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_whitelist_regex_include_pass_check_pass(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
whitelist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(whitelist_file.close)
|
|
|
|
|
whitelist_file.write('passing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path,
|
|
|
|
|
def test_tempest_run_with_include_regex_include_pass_check_pass(self):
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.include_list, path),
|
|
|
|
|
'--regex', 'passing'], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_whitelist_regex_include_fail_check_pass(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
whitelist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(whitelist_file.close)
|
|
|
|
|
whitelist_file.write('failing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--whitelist-file=%s' % path,
|
|
|
|
|
def test_tempest_run_with_include_regex_include_fail_check_pass(self):
|
|
|
|
|
path = self._get_test_list_file('failing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.include_list, path),
|
|
|
|
|
'--regex', 'pass'], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_passes_with_config_file(self):
|
|
|
|
@ -197,50 +198,75 @@ class TestRunReturnCode(base.TestCase):
|
|
|
|
|
'--config-file', self.stestr_conf_file,
|
|
|
|
|
'--regex', 'passing'], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_blacklist_failing(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
blacklist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(blacklist_file.close)
|
|
|
|
|
blacklist_file.write('failing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path], 0)
|
|
|
|
|
def test_tempest_run_with_exclude_list_failing(self):
|
|
|
|
|
path = self._get_test_list_file('failing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.exclude_list, path)], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_blacklist_passing(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
blacklist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(blacklist_file.close)
|
|
|
|
|
blacklist_file.write('passing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path], 1)
|
|
|
|
|
def test_tempest_run_with_exclude_list_passing(self):
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.exclude_list, path)], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_blacklist_regex_exclude_fail_check_pass(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
blacklist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(blacklist_file.close)
|
|
|
|
|
blacklist_file.write('failing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path,
|
|
|
|
|
def test_tempest_run_with_exclude_list_regex_exclude_fail_check_pass(self):
|
|
|
|
|
path = self._get_test_list_file('failing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.exclude_list, path),
|
|
|
|
|
'--regex', 'pass'], 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_blacklist_regex_exclude_pass_check_pass(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
blacklist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(blacklist_file.close)
|
|
|
|
|
blacklist_file.write('passing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path,
|
|
|
|
|
def test_tempest_run_with_exclude_list_regex_exclude_pass_check_pass(self):
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.exclude_list, path),
|
|
|
|
|
'--regex', 'pass'], 1)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_with_blacklist_regex_exclude_pass_check_fail(self):
|
|
|
|
|
fd, path = tempfile.mkstemp()
|
|
|
|
|
self.addCleanup(os.remove, path)
|
|
|
|
|
blacklist_file = os.fdopen(fd, 'wb', 0)
|
|
|
|
|
self.addCleanup(blacklist_file.close)
|
|
|
|
|
blacklist_file.write('passing'.encode('utf-8'))
|
|
|
|
|
self.assertRunExit(['tempest', 'run', '--blacklist-file=%s' % path,
|
|
|
|
|
def test_tempest_run_with_exclude_list_regex_exclude_pass_check_fail(self):
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self.assertRunExit(['tempest', 'run',
|
|
|
|
|
'%s=%s' % (self.exclude_list, path),
|
|
|
|
|
'--regex', 'fail'], 1)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestOldArgRunReturnCode(TestRunReturnCode):
|
|
|
|
|
"""A class for testing deprecated but still supported args.
|
|
|
|
|
|
|
|
|
|
This class will be removed once we remove the following arguments:
|
|
|
|
|
* --black-regex
|
|
|
|
|
* --blacklist-file
|
|
|
|
|
* --whitelist-file
|
|
|
|
|
"""
|
|
|
|
|
exclude_regex = '--black-regex'
|
|
|
|
|
exclude_list = '--blacklist-file'
|
|
|
|
|
include_list = '--whitelist-file'
|
|
|
|
|
|
|
|
|
|
def _test_args_passing(self, args):
|
|
|
|
|
self.assertRunExit(['tempest', 'run'] + args, 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_new_old_arg_comb(self):
|
|
|
|
|
path = self._get_test_list_file('failing')
|
|
|
|
|
self._test_args_passing(['--black-regex', 'failing',
|
|
|
|
|
'--exclude-regex', 'failing'])
|
|
|
|
|
self._test_args_passing(['--blacklist-file=' + path,
|
|
|
|
|
'--exclude-list=' + path])
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self._test_args_passing(['--whitelist-file=' + path,
|
|
|
|
|
'--include-list=' + path])
|
|
|
|
|
|
|
|
|
|
def _test_args_passing_with_stestr_repository(self, args):
|
|
|
|
|
subprocess.call(['stestr', 'init'])
|
|
|
|
|
self.assertRunExit(['tempest', 'run'] + args, 0)
|
|
|
|
|
|
|
|
|
|
def test_tempest_run_new_old_arg_comb_with_stestr_repository(self):
|
|
|
|
|
path = self._get_test_list_file('failing')
|
|
|
|
|
self._test_args_passing_with_stestr_repository(
|
|
|
|
|
['--black-regex', 'failing', '--exclude-regex', 'failing'])
|
|
|
|
|
self._test_args_passing_with_stestr_repository(
|
|
|
|
|
['--blacklist-file=' + path, '--exclude-list=' + path])
|
|
|
|
|
path = self._get_test_list_file('passing')
|
|
|
|
|
self._test_args_passing_with_stestr_repository(
|
|
|
|
|
['--whitelist-file=' + path, '--include-list=' + path])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestConfigPathCheck(base.TestCase):
|
|
|
|
|
def setUp(self):
|
|
|
|
|
super(TestConfigPathCheck, self).setUp()
|
|
|
|
|