Replace -t with --
To better support passing in arbitrary arguments to the Tempest runner, we now allow the '--' delimeter as an argument. Anything after '--' will be passed to the Tempest runner as is. This better matches how run_tempest.sh does things, and will make including test lists clearer. Closes-Bug: #1467737 Change-Id: I8e091ef7c126cbeb4c388bb19e8343efb1c4c8ab
This commit is contained in:
Paul Van Eck
parent
b782dcd148
commit
29c9ecea66
34
README.rst
34
README.rst
@ -39,36 +39,42 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
|
||||
|
||||
4. Validate your setup by running a short test.
|
||||
|
||||
`./refstack-client test -c <Path of the tempest configuration file to use> -vv -t "tempest.api.identity.admin.v2.test_roles"`
|
||||
`./refstack-client test -c <Path of the tempest configuration file to use> -vv -- tempest.api.identity.admin.v2.test_roles`
|
||||
|
||||
or
|
||||
|
||||
`./refstack-client test -c <Path of the tempest configuration file to use> -vv -t "tempest.api.identity.v2.test_token"`
|
||||
`./refstack-client test -c <Path of the tempest configuration file to use> -vv -- tempest.api.identity.v2.test_token`
|
||||
|
||||
|
||||
5. Run a full API test set.
|
||||
5. Run tests.
|
||||
|
||||
To run the entire API test set:
|
||||
|
||||
`./refstack-client test -c <Path of the tempest configuration file to use> -vv`
|
||||
|
||||
To run only those tests specified in a DefCore defined test file:
|
||||
|
||||
`./refstack-client test -c <Path of the tempest configuration file to use> -vv -- --load-list /path/to/test-list.txt
|
||||
|
||||
**Note:**
|
||||
|
||||
a. Adding -v option will show the summary output.
|
||||
b. Adding -vv option will show the Tempest test result output.
|
||||
c. Adding -t option will only test a particular test case or a test group.
|
||||
This option can be used for quick verification of the target test cases.
|
||||
For example:
|
||||
|
||||
`-t "tempest.api.identity.v2.test_token"`
|
||||
|
||||
`-t "--load-list /tmp/test-list.txt"`
|
||||
|
||||
d. Adding --upload option will have your test results be uploaded to the
|
||||
c. Adding --upload option will have your test results be uploaded to the
|
||||
default Refstack API server or the server specified by --url.
|
||||
e. Adding --url option will allow you to change where test results should
|
||||
d. Adding --url option will allow you to change where test results should
|
||||
be uploaded.
|
||||
f. Adding -r option with a string will prefix the JSON result file with the
|
||||
e. Adding -r option with a string will prefix the JSON result file with the
|
||||
given string (e.g. '-r my-test' will yield a result file like
|
||||
'my-test-0.json').
|
||||
f. Adding '--' enables you to pass arbitary arguments to the Tempest runner.
|
||||
After the first '--', all other subsequent arguments will be passed to
|
||||
the Tempest runner as is. This can be used for quick verification of the
|
||||
target test cases. For example:
|
||||
|
||||
`-- tempest.api.identity.v2.test_token`
|
||||
|
||||
`-- --load-list /tmp/test-list.txt`
|
||||
|
||||
|
||||
6. Upload test set.
|
||||
|
||||
@ -229,14 +229,14 @@ class RefstackClient:
|
||||
# Run the tempest script, specifying the conf file, the flag
|
||||
# telling it to use a virtual environment (-V), and the flag
|
||||
# telling it to run the tests serially (-t).
|
||||
cmd = (self.tempest_script, '-C', self.conf_file, '-V', '-t')
|
||||
cmd = [self.tempest_script, '-C', self.conf_file, '-V', '-t']
|
||||
|
||||
# Add the tempest test cases to test as arguments. If no test
|
||||
# cases are specified, then all Tempest API tests will be run.
|
||||
if self.args.test_cases:
|
||||
cmd += ('--', self.args.test_cases)
|
||||
if 'arbitrary_args' in self.args:
|
||||
cmd += self.args.arbitrary_args
|
||||
else:
|
||||
cmd += ('--', "tempest.api")
|
||||
cmd += ['--', "tempest.api"]
|
||||
|
||||
# If there were two verbose flags, show tempest results.
|
||||
if self.args.verbose > 1:
|
||||
@ -413,20 +413,24 @@ def parse_cli_args(args=None):
|
||||
help='Specify a string to prefix the result '
|
||||
'file with to easier distinguish them. ')
|
||||
|
||||
parser_test.add_argument('-t', '--test-cases',
|
||||
action='store',
|
||||
required=False,
|
||||
dest='test_cases',
|
||||
type=str,
|
||||
help='Specify a subset of test cases to run '
|
||||
'(e.g. --test-cases tempest.api.compute).')
|
||||
|
||||
parser_test.add_argument('-u', '--upload',
|
||||
action='store_true',
|
||||
required=False,
|
||||
help='After running Tempest, upload the test '
|
||||
'results to the default Refstack API server '
|
||||
'or the server specified by --url.')
|
||||
|
||||
# This positional argument will allow arbitrary arguments to be passed in
|
||||
# with the usage of '--'.
|
||||
parser_test.add_argument('arbitrary_args',
|
||||
nargs=argparse.REMAINDER,
|
||||
help='After the first "--", you can pass '
|
||||
'arbitrary arguments to the Tempest runner. '
|
||||
'This can be used for running specific test '
|
||||
'cases or test lists. Some examples are: '
|
||||
'-- tempest.api.compute.images.test_list_'
|
||||
'image_filters '
|
||||
'-- --load-list /tmp/test-list.txt')
|
||||
parser_test.set_defaults(func="test")
|
||||
|
||||
# List command
|
||||
|
||||
@ -53,16 +53,17 @@ class TestRefstackClient(unittest.TestCase):
|
||||
"""
|
||||
argv = [command,
|
||||
'--url', 'http://127.0.0.1', '-y']
|
||||
if command == 'test':
|
||||
argv.extend(
|
||||
('-c', kwargs.get('conf_file_name', self.conf_file_name)))
|
||||
if kwargs.get('test_cases', None):
|
||||
argv.extend(('--test-cases', kwargs.get('test_cases', None)))
|
||||
|
||||
if kwargs.get('priv_key', None):
|
||||
argv.extend(('-i', kwargs.get('priv_key', None)))
|
||||
if kwargs.get('verbose', None):
|
||||
argv.append(kwargs.get('verbose', None))
|
||||
|
||||
if command == 'test':
|
||||
argv.extend(
|
||||
('-c', kwargs.get('conf_file_name', self.conf_file_name)))
|
||||
if kwargs.get('test_cases', None):
|
||||
argv.extend(('--', kwargs.get('test_cases', None)))
|
||||
|
||||
return argv
|
||||
|
||||
def mock_keystone(self):
|
||||
@ -310,8 +311,8 @@ class TestRefstackClient(unittest.TestCase):
|
||||
client.test()
|
||||
|
||||
mock_popen.assert_called_with(
|
||||
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'),
|
||||
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'],
|
||||
stderr=None
|
||||
)
|
||||
|
||||
@ -324,7 +325,7 @@ class TestRefstackClient(unittest.TestCase):
|
||||
"""
|
||||
argv = self.mock_argv(verbose='-vv',
|
||||
test_cases='tempest.api.compute')
|
||||
argv.append('--upload')
|
||||
argv.insert(1, '--upload')
|
||||
args = rc.parse_cli_args(argv)
|
||||
client = rc.RefstackClient(args)
|
||||
client.tempest_dir = self.test_path
|
||||
@ -338,8 +339,8 @@ class TestRefstackClient(unittest.TestCase):
|
||||
client._save_json_results = MagicMock()
|
||||
client.test()
|
||||
mock_popen.assert_called_with(
|
||||
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'),
|
||||
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'],
|
||||
stderr=None
|
||||
)
|
||||
|
||||
@ -352,7 +353,7 @@ class TestRefstackClient(unittest.TestCase):
|
||||
"""
|
||||
argv = self.mock_argv(verbose='-vv', priv_key='rsa_key',
|
||||
test_cases='tempest.api.compute')
|
||||
argv.append('--upload')
|
||||
argv.insert(1, '--upload')
|
||||
args = rc.parse_cli_args(argv)
|
||||
client = rc.RefstackClient(args)
|
||||
client.tempest_dir = self.test_path
|
||||
@ -366,8 +367,8 @@ class TestRefstackClient(unittest.TestCase):
|
||||
client._save_json_results = MagicMock()
|
||||
client.test()
|
||||
mock_popen.assert_called_with(
|
||||
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'),
|
||||
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'],
|
||||
stderr=None
|
||||
)
|
||||
|
||||
@ -404,7 +405,8 @@ class TestRefstackClient(unittest.TestCase):
|
||||
"""
|
||||
argv = self.mock_argv(verbose='-vv',
|
||||
test_cases='tempest.api.compute')
|
||||
argv.extend(['--result-file-tag', 'my-test'])
|
||||
argv.insert(1, '--result-file-tag')
|
||||
argv.insert(2, 'my-test')
|
||||
args = rc.parse_cli_args(argv)
|
||||
client = rc.RefstackClient(args)
|
||||
client.tempest_dir = self.test_path
|
||||
@ -418,8 +420,8 @@ class TestRefstackClient(unittest.TestCase):
|
||||
client.test()
|
||||
|
||||
mock_popen.assert_called_with(
|
||||
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'),
|
||||
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
|
||||
'-V', '-t', '--', 'tempest.api.compute'],
|
||||
stderr=None
|
||||
)
|
||||
|
||||
@ -477,9 +479,9 @@ class TestRefstackClient(unittest.TestCase):
|
||||
client = rc.RefstackClient(args)
|
||||
self.assertRaises(SystemExit, client.upload)
|
||||
|
||||
def test_yeild_results(self):
|
||||
def test_yield_results(self):
|
||||
"""
|
||||
Test the post_results method, ensuring a requests call is made.
|
||||
Test the yield_results method, ensuring that results are retrieved.
|
||||
"""
|
||||
args = rc.parse_cli_args(self.mock_argv(command='list'))
|
||||
client = rc.RefstackClient(args)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user