diff --git a/README.rst b/README.rst index 11d949c..9d2f4b9 100644 --- a/README.rst +++ b/README.rst @@ -39,36 +39,42 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. 4. Validate your setup by running a short test. - `./refstack-client test -c -vv -t "tempest.api.identity.admin.v2.test_roles"` + `./refstack-client test -c -vv -- tempest.api.identity.admin.v2.test_roles` or - `./refstack-client test -c -vv -t "tempest.api.identity.v2.test_token"` + `./refstack-client test -c -vv -- tempest.api.identity.v2.test_token` -5. Run a full API test set. +5. Run tests. + + To run the entire API test set: `./refstack-client test -c -vv` + To run only those tests specified in a DefCore defined test file: + + `./refstack-client test -c -vv -- --load-list /path/to/test-list.txt + **Note:** a. Adding -v option will show the summary output. b. Adding -vv option will show the Tempest test result output. - c. Adding -t option will only test a particular test case or a test group. - This option can be used for quick verification of the target test cases. - For example: - - `-t "tempest.api.identity.v2.test_token"` - - `-t "--load-list /tmp/test-list.txt"` - - d. Adding --upload option will have your test results be uploaded to the + c. Adding --upload option will have your test results be uploaded to the default Refstack API server or the server specified by --url. - e. Adding --url option will allow you to change where test results should + d. Adding --url option will allow you to change where test results should be uploaded. - f. Adding -r option with a string will prefix the JSON result file with the + e. Adding -r option with a string will prefix the JSON result file with the given string (e.g. '-r my-test' will yield a result file like 'my-test-0.json'). + f. Adding '--' enables you to pass arbitary arguments to the Tempest runner. + After the first '--', all other subsequent arguments will be passed to + the Tempest runner as is. This can be used for quick verification of the + target test cases. For example: + + `-- tempest.api.identity.v2.test_token` + + `-- --load-list /tmp/test-list.txt` 6. Upload test set. diff --git a/refstack_client/refstack_client.py b/refstack_client/refstack_client.py index b5d22f3..9ed3982 100755 --- a/refstack_client/refstack_client.py +++ b/refstack_client/refstack_client.py @@ -229,14 +229,14 @@ class RefstackClient: # Run the tempest script, specifying the conf file, the flag # telling it to use a virtual environment (-V), and the flag # telling it to run the tests serially (-t). - cmd = (self.tempest_script, '-C', self.conf_file, '-V', '-t') + cmd = [self.tempest_script, '-C', self.conf_file, '-V', '-t'] # Add the tempest test cases to test as arguments. If no test # cases are specified, then all Tempest API tests will be run. - if self.args.test_cases: - cmd += ('--', self.args.test_cases) + if 'arbitrary_args' in self.args: + cmd += self.args.arbitrary_args else: - cmd += ('--', "tempest.api") + cmd += ['--', "tempest.api"] # If there were two verbose flags, show tempest results. if self.args.verbose > 1: @@ -413,20 +413,24 @@ def parse_cli_args(args=None): help='Specify a string to prefix the result ' 'file with to easier distinguish them. ') - parser_test.add_argument('-t', '--test-cases', - action='store', - required=False, - dest='test_cases', - type=str, - help='Specify a subset of test cases to run ' - '(e.g. --test-cases tempest.api.compute).') - parser_test.add_argument('-u', '--upload', action='store_true', required=False, help='After running Tempest, upload the test ' 'results to the default Refstack API server ' 'or the server specified by --url.') + + # This positional argument will allow arbitrary arguments to be passed in + # with the usage of '--'. + parser_test.add_argument('arbitrary_args', + nargs=argparse.REMAINDER, + help='After the first "--", you can pass ' + 'arbitrary arguments to the Tempest runner. ' + 'This can be used for running specific test ' + 'cases or test lists. Some examples are: ' + '-- tempest.api.compute.images.test_list_' + 'image_filters ' + '-- --load-list /tmp/test-list.txt') parser_test.set_defaults(func="test") # List command diff --git a/refstack_client/tests/unit/test_client.py b/refstack_client/tests/unit/test_client.py index 4982867..4263c25 100755 --- a/refstack_client/tests/unit/test_client.py +++ b/refstack_client/tests/unit/test_client.py @@ -53,16 +53,17 @@ class TestRefstackClient(unittest.TestCase): """ argv = [command, '--url', 'http://127.0.0.1', '-y'] - if command == 'test': - argv.extend( - ('-c', kwargs.get('conf_file_name', self.conf_file_name))) - if kwargs.get('test_cases', None): - argv.extend(('--test-cases', kwargs.get('test_cases', None))) - if kwargs.get('priv_key', None): argv.extend(('-i', kwargs.get('priv_key', None))) if kwargs.get('verbose', None): argv.append(kwargs.get('verbose', None)) + + if command == 'test': + argv.extend( + ('-c', kwargs.get('conf_file_name', self.conf_file_name))) + if kwargs.get('test_cases', None): + argv.extend(('--', kwargs.get('test_cases', None))) + return argv def mock_keystone(self): @@ -310,8 +311,8 @@ class TestRefstackClient(unittest.TestCase): client.test() mock_popen.assert_called_with( - ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'), + ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, + '-V', '-t', '--', 'tempest.api.compute'], stderr=None ) @@ -324,7 +325,7 @@ class TestRefstackClient(unittest.TestCase): """ argv = self.mock_argv(verbose='-vv', test_cases='tempest.api.compute') - argv.append('--upload') + argv.insert(1, '--upload') args = rc.parse_cli_args(argv) client = rc.RefstackClient(args) client.tempest_dir = self.test_path @@ -338,8 +339,8 @@ class TestRefstackClient(unittest.TestCase): client._save_json_results = MagicMock() client.test() mock_popen.assert_called_with( - ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'), + ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, + '-V', '-t', '--', 'tempest.api.compute'], stderr=None ) @@ -352,7 +353,7 @@ class TestRefstackClient(unittest.TestCase): """ argv = self.mock_argv(verbose='-vv', priv_key='rsa_key', test_cases='tempest.api.compute') - argv.append('--upload') + argv.insert(1, '--upload') args = rc.parse_cli_args(argv) client = rc.RefstackClient(args) client.tempest_dir = self.test_path @@ -366,8 +367,8 @@ class TestRefstackClient(unittest.TestCase): client._save_json_results = MagicMock() client.test() mock_popen.assert_called_with( - ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'), + ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, + '-V', '-t', '--', 'tempest.api.compute'], stderr=None ) @@ -404,7 +405,8 @@ class TestRefstackClient(unittest.TestCase): """ argv = self.mock_argv(verbose='-vv', test_cases='tempest.api.compute') - argv.extend(['--result-file-tag', 'my-test']) + argv.insert(1, '--result-file-tag') + argv.insert(2, 'my-test') args = rc.parse_cli_args(argv) client = rc.RefstackClient(args) client.tempest_dir = self.test_path @@ -418,8 +420,8 @@ class TestRefstackClient(unittest.TestCase): client.test() mock_popen.assert_called_with( - ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'), + ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, + '-V', '-t', '--', 'tempest.api.compute'], stderr=None ) @@ -477,9 +479,9 @@ class TestRefstackClient(unittest.TestCase): client = rc.RefstackClient(args) self.assertRaises(SystemExit, client.upload) - def test_yeild_results(self): + def test_yield_results(self): """ - Test the post_results method, ensuring a requests call is made. + Test the yield_results method, ensuring that results are retrieved. """ args = rc.parse_cli_args(self.mock_argv(command='list')) client = rc.RefstackClient(args)