From 43777f98202f56e01815755659bd6c0ef69974ed Mon Sep 17 00:00:00 2001 From: Paul Van Eck Date: Fri, 17 Jul 2015 10:13:08 -0700 Subject: [PATCH] Add argument for specifying test-list directly Because test attributes can constantly change in test lists, refstack-client should have a native argument for passing in test lists (instead of '-- --load-list') Refstack-client will normalize the tests in the given list by matching it to test IDs in the current Tempest environment. Closes-Bug: #1475407 Change-Id: I1e3f026f5cd706cf73b6febfce98cb96b742d4d6 --- README.rst | 65 +++--- refstack_client/list_parser.py | 206 ++++++++++++++++++ refstack_client/refstack_client.py | 28 ++- refstack_client/tests/unit/test-list.txt | 3 + refstack_client/tests/unit/test_client.py | 28 +++ .../tests/unit/test_list_parser.py | 186 ++++++++++++++++ requirements.txt | 3 +- 7 files changed, 486 insertions(+), 33 deletions(-) create mode 100644 refstack_client/list_parser.py create mode 100644 refstack_client/tests/unit/test-list.txt create mode 100644 refstack_client/tests/unit/test_list_parser.py diff --git a/README.rst b/README.rst index 9d2f4b9..3791b4d 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ refstack-client refstack-client is a command line utility that allows you to execute Tempest test runs based on configurations you specify. When finished running Tempest -it sends the passed test data back to the Refstack API server. +it can send the passed test data to a Refstack API server. **Environment setup** @@ -29,6 +29,7 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. 1. Prepare a tempest configuration file that is customized to your cloud environment. + 2. Go into the refstack-client directory. `cd ~/refstack-client` @@ -41,45 +42,52 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. `./refstack-client test -c -vv -- tempest.api.identity.admin.v2.test_roles` - or + or `./refstack-client test -c -vv -- tempest.api.identity.v2.test_token` 5. Run tests. - To run the entire API test set: + To run the entire API test set: `./refstack-client test -c -vv` - To run only those tests specified in a DefCore defined test file: + To run only those tests specified in a DefCore defined test file: - `./refstack-client test -c -vv -- --load-list /path/to/test-list.txt + `./refstack-client test -c -vv --test-list ` + + For example: + + `./refstack-client test -c ~/tempest.conf -vv --test-list https://raw.githubusercontent.com/openstack/defcore/master/2015.05/2015.05.required.txt` + + This will run only the test cases listed in 2015.05.required.txt. **Note:** - a. Adding -v option will show the summary output. - b. Adding -vv option will show the Tempest test result output. - c. Adding --upload option will have your test results be uploaded to the - default Refstack API server or the server specified by --url. - d. Adding --url option will allow you to change where test results should + a. Adding the `-v` option will show the summary output. + b. Adding the `-vv` option will show the Tempest test result output. + c. Adding the `--upload` option will have your test results be uploaded to the + default Refstack API server or the server specified by `--url`. + d. Adding the `--test-list` option will allow you to specify the file path or URL of + a test list text file. This test list should contain specific test cases that + should be tested. Tests lists passed in using this argument will be normalized + with the current Tempest evironment to eliminate any attribute mismatches. + e. Adding the `--url` option will allow you to change where test results should be uploaded. - e. Adding -r option with a string will prefix the JSON result file with the + f. Adding the `-r` option with a string will prefix the JSON result file with the given string (e.g. '-r my-test' will yield a result file like 'my-test-0.json'). - f. Adding '--' enables you to pass arbitary arguments to the Tempest runner. - After the first '--', all other subsequent arguments will be passed to - the Tempest runner as is. This can be used for quick verification of the - target test cases. For example: + g. Adding `--` enables you to pass arbitary arguments to the Tempest runner. + After the first `--`, all other subsequent arguments will be passed to + the Tempest runner as is. This is mainly used for quick verification of the + target test cases. (e.g. `-- tempest.api.identity.v2.test_token`) - `-- tempest.api.identity.v2.test_token` + Use `./refstack-client test --help` for the full list of arguments. - `-- --load-list /tmp/test-list.txt` +6. Upload your results. - -6. Upload test set. - - If you previously ran a test with refstack-client without the --upload + If you previously ran a test with refstack-client without the `--upload` option, you can upload your results to a Refstack API server by using the following command: @@ -87,17 +95,16 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe. The results file is a JSON file generated by refstack-client when a test has completed. This is saved in .tempest/.testrepository. When you use the - 'upload' command, you can also override the Refstack API server uploaded to - with the --url option. + `upload` command, you can also override the Refstack API server uploaded to + with the `--url` option. **Note:** - a. Adding -i option will upload test result with - digital signature. For signing refstack-client uses private RSA key. - OpenSSH format of rsa keys supported, so you can just use your ssh key - '~/.ssh/id-rsa' or generate a new one with 'ssh-keygen -b 4096'. - For now, signed test results can be considereded as private. - + a. Adding `-i ` option will upload test results with + a digital signature. For signing, refstack-client uses private RSA keys. + The OpenSSH format of RSA keys is supported, so you can just use your SSH + key '~/.ssh/id-rsa' or generate a new one with `ssh-keygen -b 4096`. + For now, signed test results can be considered private. 7. List uploaded test set. diff --git a/refstack_client/list_parser.py b/refstack_client/list_parser.py new file mode 100644 index 0000000..17e576b --- /dev/null +++ b/refstack_client/list_parser.py @@ -0,0 +1,206 @@ +# Copyright (c) 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + + +import atexit +import logging +import os +import re +import requests +import subprocess +import tempfile + + +class TestListParser(object): + + """This class is for normalizing test lists to match the tests in the + current Tempest environment. + """ + + def __init__(self, tempest_dir): + """ + Initialize the TestListParser. + + :param tempest_dir: Absolute path of the Tempest directory. + """ + self.logger = logging.getLogger(__name__) + self.tempest_dir = tempest_dir + + def _get_tempest_test_ids(self): + """This does a 'testr list-tests' on the Tempest directory in order to + get a list of full test IDs for the current Tempest environment. Test + ID mappings are then formed for these tests. + """ + cmd = (os.path.join(self.tempest_dir, 'tools/with_venv.sh'), + 'testr', 'list-tests') + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, + cwd=self.tempest_dir) + (stdout, stderr) = process.communicate() + + if process.returncode != 0: + self.logger.error(stderr) + raise subprocess.CalledProcessError(process.returncode, + ' '.join(cmd)) + + testcase_list = stdout.split('\n') + return self._form_test_id_mappings(testcase_list) + + def _form_test_id_mappings(self, test_list): + """This takes in a list of full test IDs and forms a dict containing + base test IDs mapped to their attributes. A full test ID also contains + test attributes such as '[gate,smoke]' + Ex: + 'tempest.api.test1': '[gate]' + 'tempest.api.test2': '' + 'tempest.api.test3(some_scenario)': '[smoke,gate]' + + :param test_list: List of full test IDs + """ + test_mappings = {} + for testcase in test_list: + if testcase.startswith("tempest"): + # Search for any strings like '[smoke, gate]' in the test ID. + match = re.search('(\[.*\])', testcase) + + if match: + testcase = re.sub('\[.*\]', '', testcase) + test_mappings[testcase] = match.group(1) + else: + test_mappings[testcase] = "" + return test_mappings + + def _get_base_test_ids_from_list_file(self, list_location): + """This takes in a test list file and finds all the base test IDs + for the tests listed. + Ex: + 'tempest.test1[gate,id-2]' -> 'tempest.test1' + 'tempest.test2[gate,id-3](scenario)' -> 'tempest.test2(scenario)' + + :param list_location: file path or URL location of list file + """ + try: + response = requests.get(list_location) + testcase_list = response.text.split('\n') + test_mappings = self._form_test_id_mappings(testcase_list) + # If the location isn't a valid URL, we assume it is a file path. + except requests.exceptions.MissingSchema: + try: + with open(list_location) as data_file: + testcase_list = [line.rstrip('\n') for line in data_file] + test_mappings = self._form_test_id_mappings(testcase_list) + except Exception: + self.logger.error("Error reading the passed in test list " + + "file.") + raise + except Exception: + self.logger.error("Error reading the passed in test list file.") + raise + + return list(test_mappings.keys()) + + def _get_full_test_ids(self, tempest_ids, base_ids): + """This will remake the test ID list with the full IDs of the current + Tempest environment. The Tempest test ID dict should have the correct + mappings. + + :param tempest_ids: dict containing test ID mappings + :param base_ids: list containing base test IDs + """ + test_list = [] + for test_id in base_ids: + try: + attr = tempest_ids[test_id] + # If the test has a scenario in the test ID, but also has some + # additional attributes, the attributes need to go before the + # scenario. + if '(' in test_id and attr: + components = test_id.split('(', 1) + test_portion = components[0] + scenario = "(" + components[1] + test_list.append(test_portion + attr + scenario) + else: + test_list.append(test_id + attr) + except KeyError: + self.logger.warning("Test %s not found in Tempest list." % + test_id) + self.logger.debug("Number of tests: " + str(len(test_list))) + return test_list + + def _write_normalized_test_list(self, test_ids): + """Create a temporary file to pass into testr containing a list of test + IDs that should be tested. + + :param test_ids: list of full test IDs + """ + temp = tempfile.NamedTemporaryFile(delete=False) + for test_id in test_ids: + temp.write("%s\n" % test_id) + temp.flush() + + # Register the created file for cleanup. + atexit.register(self._remove_test_list_file, temp.name) + return temp.name + + def _remove_test_list_file(self, file_path): + """Delete the given file. + + :param file_path: string containing the location of the file + """ + if os.path.isfile(file_path): + os.remove(file_path) + + def setup_venv(self, log_level): + """If for some reason the virtualenv for Tempest has not been + set up, then install it. This is to ensure that 'testr list-tests' + works. + + :param log_level: integer denoting the log level (e.g. logging.DEBUG) + """ + if not os.path.isdir(os.path.join(self.tempest_dir, ".venv")): + self.logger.info("Installing Tempest virtualenv. This may take " + "a while.") + cmd = ('python', + os.path.join(self.tempest_dir, "tools/install_venv.py")) + + # Only show installation messages if the logging level is DEBUG. + if log_level <= logging.DEBUG: + stdout = None + else: + stdout = open(os.devnull, 'w') + + process = subprocess.Popen(cmd, cwd=self.tempest_dir, + stdout=stdout) + process.communicate() + + if process.returncode != 0: + self.logger.error("Error installing Tempest virtualenv.") + raise subprocess.CalledProcessError(process.returncode, + ' '.join(cmd)) + + def get_normalized_test_list(self, list_location): + """This will take in the user's test list and will normalize it + so that the test cases in the list map to actual full test IDS in + the Tempest environment. + + :param list_location: file path or URL of the test list + """ + tempest_test_ids = self._get_tempest_test_ids() + if not tempest_test_ids: + return None + base_test_ids = self._get_base_test_ids_from_list_file(list_location) + full_capability_test_ids = self._get_full_test_ids(tempest_test_ids, + base_test_ids) + list_file = self._write_normalized_test_list(full_capability_test_ids) + return list_file diff --git a/refstack_client/refstack_client.py b/refstack_client/refstack_client.py index 9ed3982..c1bc6d1 100755 --- a/refstack_client/refstack_client.py +++ b/refstack_client/refstack_client.py @@ -42,6 +42,7 @@ import requests import requests.exceptions import six.moves from subunit_processor import SubunitProcessor +from list_parser import TestListParser def get_input(): @@ -231,9 +232,20 @@ class RefstackClient: # telling it to run the tests serially (-t). cmd = [self.tempest_script, '-C', self.conf_file, '-V', '-t'] - # Add the tempest test cases to test as arguments. If no test - # cases are specified, then all Tempest API tests will be run. - if 'arbitrary_args' in self.args: + # If a test list was specified, have it take precedence. + if self.args.test_list: + self.logger.info("Normalizing test list...") + parser = TestListParser(os.path.abspath(self.tempest_dir)) + parser.setup_venv(self.logger.getEffectiveLevel()) + list_file = parser.get_normalized_test_list(self.args.test_list) + if list_file: + cmd += ('--', '--load-list', list_file) + else: + self.logger.error("Error normalizing passed in test list.") + exit(1) + elif 'arbitrary_args' in self.args: + # Add the tempest test cases to test as arguments. If no test + # cases are specified, then all Tempest API tests will be run. cmd += self.args.arbitrary_args else: cmd += ['--', "tempest.api"] @@ -413,6 +425,16 @@ def parse_cli_args(args=None): help='Specify a string to prefix the result ' 'file with to easier distinguish them. ') + parser_test.add_argument('--test-list', + action='store', + required=False, + dest='test_list', + type=str, + help='Specify the file path or URL of a test ' + 'list text file. This test list will ' + 'contain specific test cases that should ' + 'be tested.') + parser_test.add_argument('-u', '--upload', action='store_true', required=False, diff --git a/refstack_client/tests/unit/test-list.txt b/refstack_client/tests/unit/test-list.txt new file mode 100644 index 0000000..4a8c57b --- /dev/null +++ b/refstack_client/tests/unit/test-list.txt @@ -0,0 +1,3 @@ +tempest.api.test1[gate] +tempest.api.test2 +tempest.api.test3[foo,bar](scenario) diff --git a/refstack_client/tests/unit/test_client.py b/refstack_client/tests/unit/test_client.py index 4263c25..8dac226 100755 --- a/refstack_client/tests/unit/test_client.py +++ b/refstack_client/tests/unit/test_client.py @@ -26,6 +26,7 @@ import unittest import refstack_client.refstack_client as rc +import refstack_client.list_parser as lp class TestRefstackClient(unittest.TestCase): @@ -381,6 +382,33 @@ class TestRefstackClient(unittest.TestCase): sign_with='rsa_key' ) + def test_run_tempest_with_test_list(self): + """Test that the Tempest script runs with a test list file.""" + argv = self.mock_argv(verbose='-vv') + argv.extend(['--test-list', 'test-list.txt']) + args = rc.parse_cli_args(argv) + client = rc.RefstackClient(args) + client.tempest_dir = self.test_path + mock_popen = self.patch( + 'refstack_client.refstack_client.subprocess.Popen', + return_value=MagicMock(returncode=0)) + self.patch("os.path.isfile", return_value=True) + self.mock_keystone() + client.get_passed_tests = MagicMock(return_value=[{'name': 'test'}]) + client._save_json_results = MagicMock() + client.post_results = MagicMock() + lp.TestListParser.get_normalized_test_list = MagicMock( + return_value="/tmp/some-list") + client.test() + + lp.TestListParser.get_normalized_test_list.assert_called_with( + 'test-list.txt') + mock_popen.assert_called_with( + ['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, + '-V', '-t', '--', '--load-list', '/tmp/some-list'], + stderr=None + ) + def test_run_tempest_no_conf_file(self): """ Test when a nonexistent configuration file is passed in. diff --git a/refstack_client/tests/unit/test_list_parser.py b/refstack_client/tests/unit/test_list_parser.py new file mode 100644 index 0000000..3568593 --- /dev/null +++ b/refstack_client/tests/unit/test_list_parser.py @@ -0,0 +1,186 @@ +# Copyright 2015 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +import logging +import os +import requests +import subprocess + +import httmock +import mock +import unittest + +import refstack_client.list_parser as parser + + +class TestTestListParser(unittest.TestCase): + + test_path = os.path.dirname(os.path.realpath(__file__)) + tempest_dir = "some_dir/.tempest" + + def setUp(self): + """Test case setup""" + logging.disable(logging.CRITICAL) + self.parser = parser.TestListParser(self.tempest_dir) + + def test_get_tempest_test_ids(self): + """Test that the tempest test-list is correctly parsed.""" + test_list = ("tempest.test.one[gate]\n" + "tempest.test.two[gate,smoke]\n" + "tempest.test.three(scenario)\n" + "tempest.test.four[gate](another_scenario)\n" + "tempest.test.five") + process_mock = mock.Mock(returncode=0) + attrs = {'communicate.return_value': (test_list, None)} + process_mock.configure_mock(**attrs) + subprocess.Popen = mock.Mock(return_value=process_mock) + output = self.parser._get_tempest_test_ids() + + subprocess.Popen.assert_called_with( + ("%s/tools/with_venv.sh" % self.tempest_dir, "testr", + "list-tests"), + stdout=subprocess.PIPE, + cwd=self.tempest_dir) + expected_output = {"tempest.test.one": "[gate]", + "tempest.test.two": "[gate,smoke]", + "tempest.test.three(scenario)": "", + "tempest.test.four(another_scenario)": "[gate]", + "tempest.test.five": ""} + self.assertEqual(expected_output, output) + + def test_get_tempest_test_ids_fail(self): + """Test when the test listing subprocess returns a non-zero exit + status. + """ + process_mock = mock.Mock(returncode=1) + attrs = {'communicate.return_value': (mock.ANY, None)} + process_mock.configure_mock(**attrs) + subprocess.Popen = mock.Mock(return_value=process_mock) + with self.assertRaises(subprocess.CalledProcessError): + self.parser._get_tempest_test_ids() + + def test_form_test_id_mappings(self): + """Test the test ID to attribute dict builder function.""" + test_list = ["tempest.test.one[gate]", + "tempest.test.two[gate,smoke]", + "tempest.test.three(scenario)", + "tempest.test.four[gate](another_scenario)", + "tempest.test.five"] + + expected_output = {"tempest.test.one": "[gate]", + "tempest.test.two": "[gate,smoke]", + "tempest.test.three(scenario)": "", + "tempest.test.four(another_scenario)": "[gate]", + "tempest.test.five": ""} + output = self.parser._form_test_id_mappings(test_list) + self.assertEqual(expected_output, output) + + def test_get_base_test_ids_from_list_file(self): + """test that we can get the base test IDs from a test list file.""" + list_file = self.test_path + "/test-list.txt" + test_list = self.parser._get_base_test_ids_from_list_file(list_file) + expected_list = ['tempest.api.test1', + 'tempest.api.test2', + 'tempest.api.test3(scenario)'] + self.assertEqual(expected_list, sorted(test_list)) + + def test_get_base_test_ids_from_list_files_invalid_file(self): + """Test that we get an exception when passing in a nonexistent file.""" + some_file = self.test_path + "/nonexistent.json" + with self.assertRaises(Exception): + self.parser._get_base_test_ids_from_list_file(some_file) + + def test_get_base_test_ids_from_list_file_url(self): + """Test that we can parse the test cases from a test list URL.""" + list_file = self.test_path + "/test-list.txt" + + with open(list_file, 'rb') as f: + content = f.read() + + @httmock.all_requests + def request_mock(url, request): + return {'status_code': 200, + 'content': content} + + with httmock.HTTMock(request_mock): + online_list = self.parser._get_base_test_ids_from_list_file( + "http://127.0.0.1/test-list.txt") + + expected_list = ['tempest.api.test1', + 'tempest.api.test2', + 'tempest.api.test3(scenario)'] + self.assertEqual(expected_list, sorted(online_list)) + + def test_get_base_test_ids_from_list_file_invalid_url(self): + """Test a case of an invalid URL schema.""" + with self.assertRaises(requests.exceptions.RequestException): + self.parser._get_base_test_ids_from_list_file("foo://sasas.com") + + def test_get_full_test_ids(self): + """Test that full test IDs can be formed.""" + tempest_ids = {"tempest.test.one": "[gate]", + "tempest.test.two": "[gate,smoke]", + "tempest.test.three(scenario)": "", + "tempest.test.four(another_scenario)": "[gate]", + "tempest.test.five": ""} + + base_ids = ["tempest.test.one", + "tempest.test.four(another_scenario)", + "tempest.test.five"] + + output_list = self.parser._get_full_test_ids(tempest_ids, base_ids) + expected_list = ["tempest.test.one[gate]", + "tempest.test.four[gate](another_scenario)", + "tempest.test.five"] + self.assertEqual(expected_list, output_list) + + def test_get_full_test_ids_with_nonexistent_test(self): + """Test when a test ID doesn't exist in the Tempest environment.""" + tempest_ids = {"tempest.test.one": "[gate]", + "tempest.test.two": "[gate,smoke]"} + base_ids = ["tempest.test.one", "tempest.test.foo"] + output_list = self.parser._get_full_test_ids(tempest_ids, base_ids) + + self.assertEqual(["tempest.test.one[gate]"], output_list) + + def test_write_normalized_test_list(self): + """Test that a normalized test list is written to disk.""" + test_ids = ["tempest.test.one[gate]", "tempest.test.five"] + test_file = self.parser._write_normalized_test_list(test_ids) + + # Check that the tempest IDs in the file match the expected test + # ID list. + with open(test_file, 'rb') as f: + file_contents = f.read() + testcase_list = filter(None, file_contents.split('\n')) + + self.assertEqual(test_ids, testcase_list) + + def test_setup_venv(self): + """Test whether the proper script is called to setup a virtualenv.""" + process_mock = mock.Mock(returncode=0) + subprocess.Popen = mock.Mock(return_value=process_mock) + self.parser.setup_venv(logging.DEBUG) + subprocess.Popen.assert_called_with( + ("python", "%s/tools/install_venv.py" % self.tempest_dir), + cwd=self.tempest_dir, + stdout=None) + + def test_setup_venv_fail(self): + """Test whether the proper script is called to setup a virtualenv.""" + process_mock = mock.Mock(returncode=1) + subprocess.Popen = mock.Mock(return_value=process_mock) + with self.assertRaises(subprocess.CalledProcessError): + self.parser.setup_venv(logging.DEBUG) diff --git a/requirements.txt b/requirements.txt index 2bdf036..ae29239 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ python-keystoneclient>=0.10.0 gitpython>=0.3.2.RC1 python-subunit>=0.0.18 -pycrypto>=2.6.1 \ No newline at end of file +pycrypto>=2.6.1 +requests>=2.5.2