@ -68,6 +68,11 @@ class TestTempestRun(base.TestCase):
class TestRunReturnCode ( base . TestCase ) :
exclude_regex = ' --exclude-regex '
exclude_list = ' --exclude-list '
include_list = ' --include-list '
def setUp ( self ) :
super ( TestRunReturnCode , self ) . setUp ( )
# Setup test dirs
@ -92,6 +97,14 @@ class TestRunReturnCode(base.TestCase):
self . addCleanup ( os . chdir , os . path . abspath ( os . curdir ) )
os . chdir ( self . directory )
def _get_test_list_file ( self , content ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
test_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( test_file . close )
test_file . write ( content . encode ( ' utf-8 ' ) )
return path
def assertRunExit ( self , cmd , expected ) :
p = subprocess . Popen ( cmd , stdout = subprocess . PIPE ,
stderr = subprocess . PIPE )
@ -115,19 +128,23 @@ class TestRunReturnCode(base.TestCase):
subprocess . call ( [ ' stestr ' , ' init ' ] )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --regex ' , ' failing ' ] , 1 )
def test_tempest_run_blackregex_failing ( self ) :
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --black-regex ' , ' failing ' ] , 0 )
def test_tempest_run_exclude_regex_failing ( self ) :
self . assertRunExit ( [ ' tempest ' , ' run ' ,
self . exclude_regex , ' failing ' ] , 0 )
def test_tempest_run_ black regex_failing_with_stestr_repository( self ) :
def test_tempest_run_ exclude_ regex_failing_with_stestr_repository( self ) :
subprocess . call ( [ ' stestr ' , ' init ' ] )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --black-regex ' , ' failing ' ] , 0 )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
self . exclude_regex , ' failing ' ] , 0 )
def test_tempest_run_blackregex_passing ( self ) :
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --black-regex ' , ' passing ' ] , 1 )
def test_tempest_run_exclude_regex_passing ( self ) :
self . assertRunExit ( [ ' tempest ' , ' run ' ,
self . exclude_regex , ' passing ' ] , 1 )
def test_tempest_run_ black regex_passing_with_stestr_repository( self ) :
def test_tempest_run_ exclude_ regex_passing_with_stestr_repository( self ) :
subprocess . call ( [ ' stestr ' , ' init ' ] )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --black-regex ' , ' passing ' ] , 1 )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
self . exclude_regex , ' passing ' ] , 1 )
def test_tempest_run_fails ( self ) :
self . assertRunExit ( [ ' tempest ' , ' run ' ] , 1 )
@ -149,47 +166,31 @@ class TestRunReturnCode(base.TestCase):
self . assertEqual ( result , tests )
def test_tempest_run_with_worker_file ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
worker_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( worker_file . close )
worker_file . write (
' - worker: \n - passing \n concurrency: 3 ' . encode ( ' utf-8 ' ) )
path = self . _get_test_list_file (
' - worker: \n - passing \n concurrency: 3 ' )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --worker-file= %s ' % path ] , 0 )
def test_tempest_run_with_whitelist ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
whitelist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( whitelist_file . close )
whitelist_file . write ( ' passing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --whitelist-file= %s ' % path ] , 0 )
def test_tempest_run_with_include_list ( self ) :
path = self . _get_test_list_file ( ' passing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . include_list , path ) ] , 0 )
def test_tempest_run_with_whitelist_regex_include_pass_check_fail ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
whitelist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( whitelist_file . close )
whitelist_file . write ( ' passing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --whitelist-file= %s ' % path ,
def test_tempest_run_with_include_regex_include_pass_check_fail ( self ) :
path = self . _get_test_list_file ( ' passing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . include_list , path ) ,
' --regex ' , ' fail ' ] , 1 )
def test_tempest_run_with_whitelist_regex_include_pass_check_pass ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
whitelist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( whitelist_file . close )
whitelist_file . write ( ' passing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --whitelist-file= %s ' % path ,
def test_tempest_run_with_include_regex_include_pass_check_pass ( self ) :
path = self . _get_test_list_file ( ' passing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . include_list , path ) ,
' --regex ' , ' passing ' ] , 0 )
def test_tempest_run_with_whitelist_regex_include_fail_check_pass ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
whitelist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( whitelist_file . close )
whitelist_file . write ( ' failing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --whitelist-file= %s ' % path ,
def test_tempest_run_with_include_regex_include_fail_check_pass ( self ) :
path = self . _get_test_list_file ( ' failing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . include_list , path ) ,
' --regex ' , ' pass ' ] , 1 )
def test_tempest_run_passes_with_config_file ( self ) :
@ -197,50 +198,75 @@ class TestRunReturnCode(base.TestCase):
' --config-file ' , self . stestr_conf_file ,
' --regex ' , ' passing ' ] , 0 )
def test_tempest_run_with_blacklist_failing ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
blacklist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( blacklist_file . close )
blacklist_file . write ( ' failing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --blacklist-file= %s ' % path ] , 0 )
def test_tempest_run_with_exclude_list_failing ( self ) :
path = self . _get_test_list_file ( ' failing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . exclude_list , path ) ] , 0 )
def test_tempest_run_with_blacklist_passing ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
blacklist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( blacklist_file . close )
blacklist_file . write ( ' passing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --blacklist-file= %s ' % path ] , 1 )
def test_tempest_run_with_exclude_list_passing ( self ) :
path = self . _get_test_list_file ( ' passing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . exclude_list , path ) ] , 1 )
def test_tempest_run_with_blacklist_regex_exclude_fail_check_pass ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
blacklist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( blacklist_file . close )
blacklist_file . write ( ' failing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --blacklist-file= %s ' % path ,
def test_tempest_run_with_exclude_list_regex_exclude_fail_check_pass ( self ) :
path = self . _get_test_list_file ( ' failing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . exclude_list , path ) ,
' --regex ' , ' pass ' ] , 0 )
def test_tempest_run_with_blacklist_regex_exclude_pass_check_pass ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
blacklist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( blacklist_file . close )
blacklist_file . write ( ' passing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --blacklist-file= %s ' % path ,
def test_tempest_run_with_exclude_list_regex_exclude_pass_check_pass ( self ) :
path = self . _get_test_list_file ( ' passing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . exclude_list , path ) ,
' --regex ' , ' pass ' ] , 1 )
def test_tempest_run_with_blacklist_regex_exclude_pass_check_fail ( self ) :
fd , path = tempfile . mkstemp ( )
self . addCleanup ( os . remove , path )
blacklist_file = os . fdopen ( fd , ' wb ' , 0 )
self . addCleanup ( blacklist_file . close )
blacklist_file . write ( ' passing ' . encode ( ' utf-8 ' ) )
self . assertRunExit ( [ ' tempest ' , ' run ' , ' --blacklist-file= %s ' % path ,
def test_tempest_run_with_exclude_list_regex_exclude_pass_check_fail ( self ) :
path = self . _get_test_list_file ( ' passing ' )
self . assertRunExit ( [ ' tempest ' , ' run ' ,
' %s = %s ' % ( self . exclude_list , path ) ,
' --regex ' , ' fail ' ] , 1 )
class TestOldArgRunReturnCode ( TestRunReturnCode ) :
""" A class for testing deprecated but still supported args.
This class will be removed once we remove the following arguments :
* - - black - regex
* - - blacklist - file
* - - whitelist - file
"""
exclude_regex = ' --black-regex '
exclude_list = ' --blacklist-file '
include_list = ' --whitelist-file '
def _test_args_passing ( self , args ) :
self . assertRunExit ( [ ' tempest ' , ' run ' ] + args , 0 )
def test_tempest_run_new_old_arg_comb ( self ) :
path = self . _get_test_list_file ( ' failing ' )
self . _test_args_passing ( [ ' --black-regex ' , ' failing ' ,
' --exclude-regex ' , ' failing ' ] )
self . _test_args_passing ( [ ' --blacklist-file= ' + path ,
' --exclude-list= ' + path ] )
path = self . _get_test_list_file ( ' passing ' )
self . _test_args_passing ( [ ' --whitelist-file= ' + path ,
' --include-list= ' + path ] )
def _test_args_passing_with_stestr_repository ( self , args ) :
subprocess . call ( [ ' stestr ' , ' init ' ] )
self . assertRunExit ( [ ' tempest ' , ' run ' ] + args , 0 )
def test_tempest_run_new_old_arg_comb_with_stestr_repository ( self ) :
path = self . _get_test_list_file ( ' failing ' )
self . _test_args_passing_with_stestr_repository (
[ ' --black-regex ' , ' failing ' , ' --exclude-regex ' , ' failing ' ] )
self . _test_args_passing_with_stestr_repository (
[ ' --blacklist-file= ' + path , ' --exclude-list= ' + path ] )
path = self . _get_test_list_file ( ' passing ' )
self . _test_args_passing_with_stestr_repository (
[ ' --whitelist-file= ' + path , ' --include-list= ' + path ] )
class TestConfigPathCheck ( base . TestCase ) :
def setUp ( self ) :
super ( TestConfigPathCheck , self ) . setUp ( )