From e6104925376ddf592ddcc03694697a98ad23fb27 Mon Sep 17 00:00:00 2001 From: Luz Cazares Date: Tue, 14 Jun 2016 14:28:01 -0700 Subject: [PATCH] Replace run_tempest.sh script with ostestr command QA community default test runner is ostestr, and using run_tempest.sh script is no longer recommended. BP replaces run_tempest.sh script with direct ostestr call passing the appropriate parameters. Implements blueprint: implement-ostestr-refstackclient Change-Id: If4ffed70c8320abb4b4c7bfd1b84ca7892272665 --- README.rst | 20 ++++--- refstack_client/list_parser.py | 15 ++++++ refstack_client/refstack_client.py | 64 ++++++++++++++--------- refstack_client/tests/unit/test_client.py | 34 ++++++------ 4 files changed, 82 insertions(+), 51 deletions(-) diff --git a/README.rst b/README.rst index 7df15f2..8b60c5d 100644 --- a/README.rst +++ b/README.rst @@ -30,6 +30,10 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. 1. Prepare a tempest configuration file that is customized to your cloud environment. + Note: Use Tempest Pre-Provisioned credentials_ to provide user test accounts. :: + +.. _credentials: http://docs.openstack.org/developer/tempest/configuration.html#pre-provisioned-credentials + 2. Go into the refstack-client directory:: cd ~/refstack-client @@ -40,11 +44,11 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. 4. Validate your setup by running a short test:: - ./refstack-client test -c -v -- tempest.api.identity.admin.v2.test_roles + ./refstack-client test -c -v -- --regex tempest.api.identity.admin.v2.test_roles or :: - ./refstack-client test -c -v -- tempest.api.identity.v2.test_token + ./refstack-client test -c -v -- --regex tempest.api.identity.v2.test_token 5. Run tests. @@ -79,10 +83,10 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. e. Adding the ``-r`` option with a string will prefix the JSON result file with the given string (e.g. ``-r my-test`` will yield a result file like 'my-test-0.json'). - f. Adding ``--`` enables you to pass arbitary arguments to the Tempest runner. + f. Adding ``--`` enables you to pass arbitrary arguments to the ostestr runner. After the first ``--``, all other subsequent arguments will be passed to - the Tempest runner as is. This is mainly used for quick verification of the - target test cases. (e.g. ``-- tempest.api.identity.v2.test_token``) + the ostestr runner as is. This is mainly used for quick verification of the + target test cases. (e.g. ``-- --regex tempest.api.identity.v2.test_token``) Use ``./refstack-client test --help`` for the full list of arguments. @@ -129,8 +133,8 @@ configuration, you can activate a working Tempest environment by switching to that directory and using the installed dependencies. 1. ``cd .tempest`` -2. run tempest with ``./run_tempest.sh -V`` or ``source ./.venv/bin/activate`` - and run tests manually with ``testr``. +2. ``source ./.venv/bin/activate`` + and run tests manually with ``ostestr``. This will make the entire Tempest environment available for you to run, -including the ``run_tempest`` script and ``testr``. +including the ``ostestr`` and ``testr`` commands. diff --git a/refstack_client/list_parser.py b/refstack_client/list_parser.py index e3499a0..ab01e5c 100644 --- a/refstack_client/list_parser.py +++ b/refstack_client/list_parser.py @@ -205,3 +205,18 @@ class TestListParser(object): base_test_ids) list_file = self._write_normalized_test_list(full_capability_test_ids) return list_file + + def create_whitelist(self, list_location): + """This takes in a test list file, get normalized, and get whitelist + test IDs. + Ex: + 'tempest.test1[id-2,gate]' -> id-2 + 'tempest.test2[id-3,smoke](scenario)' -> id-3 + + :param list_location: file path or URL location of list file + """ + normalized_list = open(self.get_normalized_test_list(list_location), + 'r').read() + # Keep the IDs + test_ids = re.sub("[\,\]].*", "", re.sub(".*.\[", "", normalized_list)) + return self._write_normalized_test_list(test_ids.split('\n')) diff --git a/refstack_client/refstack_client.py b/refstack_client/refstack_client.py index 17118f7..6895d8d 100755 --- a/refstack_client/refstack_client.py +++ b/refstack_client/refstack_client.py @@ -73,7 +73,9 @@ class RefstackClient: self.logger.addHandler(self.console_log_handle) self.args = args - self.tempest_dir = '.tempest' + self.current_dir = os.path.dirname(os.path.realpath(__file__)) + self.refstack_dir = os.path.dirname(self.current_dir) + self.tempest_dir = os.path.join(self.refstack_dir, '.tempest') # set default log level to INFO. if self.args.silent: @@ -102,14 +104,9 @@ class RefstackClient: "does not exist: %s" % self.tempest_dir) exit(1) - self.tempest_script = os.path.join(self.tempest_dir, - 'run_tempest.sh') - self.conf_file = self.args.conf_file self.conf = ConfigParser.SafeConfigParser() self.conf.read(self.args.conf_file) - self.tempest_script = os.path.join(self.tempest_dir, - 'run_tempest.sh') def _prep_upload(self): '''Prepare an upload to the RefStack_api''' @@ -406,28 +403,32 @@ class RefstackClient: self.logger.info("Starting Tempest test...") start_time = time.time() - # Run the tempest script, specifying the conf file, the flag - # telling it to use a virtual environment (-V), and the flag - # telling it to run the tests serially (-t). - cmd = [self.tempest_script, '-C', self.conf_file, '-V', '-t'] - + # Run the ostestr command, conf file specified at _prep_test method + # Use virtual environment (wrapper script) + # telling it to run the tests serially (--serial). + wrapper = os.path.join(self.tempest_dir, 'tools', 'with_venv.sh') + cmd = [wrapper, 'ostestr', '--serial', '--no-slowest'] # If a test list was specified, have it take precedence. if self.args.test_list: self.logger.info("Normalizing test list...") parser = TestListParser(os.path.abspath(self.tempest_dir)) parser.setup_venv(self.logger.getEffectiveLevel()) - list_file = parser.get_normalized_test_list(self.args.test_list) + # get whitelist + list_file = parser.create_whitelist(self.args.test_list) if list_file: - cmd += ('--', '--load-list', list_file) + cmd += ('--whitelist_file', list_file) else: self.logger.error("Error normalizing passed in test list.") exit(1) elif 'arbitrary_args' in self.args: - # Add the tempest test cases to test as arguments. If no test - # cases are specified, then all Tempest API tests will be run. - cmd += self.args.arbitrary_args + # Additional arguments for ostestr runner + # otherwise run all Tempest API tests. + # keep usage(-- testCaseName) + tmp = self.args.arbitrary_args[1:] + if tmp: + cmd += (tmp if tmp[0].startswith('-') else ['--regex'] + tmp) else: - cmd += ['--', "tempest.api"] + cmd += ['--regex', "tempest.api"] # If there were two verbose flags, show tempest results. if self.args.verbose > 0: @@ -437,12 +438,14 @@ class RefstackClient: # results to stderr. stderr = open(os.devnull, 'w') - # Execute the tempest test script in a subprocess. + # Execute the ostestr command in a subprocess. + os.chdir(self.tempest_dir) process = subprocess.Popen(cmd, stderr=stderr) process.communicate() + os.chdir(self.refstack_dir) - # If the subunit file was created, then the Tempest test was at least - # started successfully. + # If the subunit file was created, then test cases were executed via + # ostestr and there is test output to process. if os.path.isfile(results_file): end_time = time.time() elapsed = end_time - start_time @@ -472,8 +475,17 @@ class RefstackClient: self.post_results(self.args.url, content, sign_with=self.args.priv_key) else: - self.logger.error("Problem executing Tempest script. Exit code %d", - process.returncode) + msg1 = ("ostestr command did not generate a results file under " + "the Refstack .tempest/.testrepository directory." + "Review command and try again.") + msg2 = ("Problem executing ostestr command. Results file not " + "generated hence no file to upload. " + "Review arbitrary arguments.") + if process.returncode != 0: + self.logger.warning(msg1) + if self.args.upload: + self.logger.error(msg2) + return process.returncode def upload(self): @@ -704,12 +716,12 @@ def parse_cli_args(args=None): parser_test.add_argument('arbitrary_args', nargs=argparse.REMAINDER, help='After the first "--", you can pass ' - 'arbitrary arguments to the Tempest runner. ' + 'arbitrary arguments to the ostestr runner. ' 'This can be used for running specific test ' 'cases or test lists. Some examples are: ' - '-- tempest.api.compute.images.test_list_' - 'image_filters ' - '-- --load-list /tmp/test-list.txt') + '-- --regex tempest.api.compute.images.' + 'test_list_image_filters OR ' + '-- --whitelist_file /tmp/testid-list.txt') parser_test.set_defaults(func="test") # List command diff --git a/refstack_client/tests/unit/test_client.py b/refstack_client/tests/unit/test_client.py index de142dc..80c2cb0 100755 --- a/refstack_client/tests/unit/test_client.py +++ b/refstack_client/tests/unit/test_client.py @@ -569,8 +569,8 @@ class TestRefstackClient(unittest.TestCase): client.test() mock_popen.assert_called_with( - ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'], + ['%s/tools/with_venv.sh' % self.test_path, 'ostestr', + '--serial', '--no-slowest', '--regex', 'tempest.api.compute'], stderr=None ) @@ -600,8 +600,8 @@ class TestRefstackClient(unittest.TestCase): client._get_cpid_from_keystone = MagicMock() client.test() mock_popen.assert_called_with( - ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'], + ['%s/tools/with_venv.sh' % self.test_path, 'ostestr', + '--serial', '--no-slowest', '--regex', 'tempest.api.compute'], stderr=None ) @@ -633,8 +633,8 @@ class TestRefstackClient(unittest.TestCase): return_value='test-id') client.test() mock_popen.assert_called_with( - ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'], + ['%s/tools/with_venv.sh' % self.test_path, 'ostestr', + '--serial', '--no-slowest', '--regex', 'tempest.api.compute'], stderr=None ) @@ -664,15 +664,16 @@ class TestRefstackClient(unittest.TestCase): client.post_results = MagicMock() lp.TestListParser.get_normalized_test_list = MagicMock( return_value="/tmp/some-list") + lp.TestListParser.create_whitelist = MagicMock( + return_value="/tmp/some-list") client._get_keystone_config = MagicMock( return_value=self.v2_config) client.test() - lp.TestListParser.get_normalized_test_list.assert_called_with( - 'test-list.txt') + lp.TestListParser.create_whitelist.assert_called_with('test-list.txt') mock_popen.assert_called_with( - ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', '--load-list', '/tmp/some-list'], + ['%s/tools/with_venv.sh' % self.test_path, 'ostestr', '--serial', + '--no-slowest', '--whitelist_file', '/tmp/some-list'], stderr=None ) @@ -719,15 +720,14 @@ class TestRefstackClient(unittest.TestCase): client.test() mock_popen.assert_called_with( - ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, - '-V', '-t', '--', 'tempest.api.compute'], + ['%s/tools/with_venv.sh' % self.test_path, 'ostestr', + '--serial', '--no-slowest', '--regex', 'tempest.api.compute'], stderr=None ) - - directory = os.path.dirname(os.path.realpath(__file__)) # Since '1' is in the next-stream file, we expect the JSON output file # to be 'my-test-1.json'. - expected_file = directory + "/.testrepository/my-test-1.json" + expected_file = os.path.join(self.test_path, '.testrepository', + 'my-test-1.json') client._save_json_results.assert_called_with(mock.ANY, expected_file) def test_failed_run(self): @@ -740,12 +740,12 @@ class TestRefstackClient(unittest.TestCase): client = rc.RefstackClient(args) client.tempest_dir = self.test_path self.mock_data() - client.logger.error = MagicMock() + client.logger.warning = MagicMock() client._get_keystone_config = MagicMock( return_value=self.v2_config) client._get_cpid_from_keystone = MagicMock() client.test() - self.assertTrue(client.logger.error.called) + self.assertTrue(client.logger.warning.called) def test_upload(self): """