openinfra/refstack-client
Code Issues Proposed changes

Replace run_tempest.sh script with ostestr command

Browse Source 
QA community default test runner is ostestr, and using run_tempest.sh
script is no longer recommended. BP replaces run_tempest.sh script
with direct ostestr call passing the appropriate parameters.

Implements blueprint: implement-ostestr-refstackclient

Change-Id: If4ffed70c8320abb4b4c7bfd1b84ca7892272665
This commit is contained in:
Luz Cazares 2016-06-14 14:28:01 -07:00
parent 648220f054
commit e610492537
4 changed files with 82 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
README.rst
View File

@ -30,6 +30,10 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
1. Prepare a tempest configuration file that is customized to your cloud 1. Prepare a tempest configuration file that is customized to your cloud
environment. environment.
Note: Use Tempest Pre-Provisioned credentials_ to provide user test accounts. ::
.. _credentials: http://docs.openstack.org/developer/tempest/configuration.html#pre-provisioned-credentials
2. Go into the refstack-client directory:: 2. Go into the refstack-client directory::
cd ~/refstack-client cd ~/refstack-client
@ -40,11 +44,11 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
4. Validate your setup by running a short test:: 4. Validate your setup by running a short test::
./refstack-client test -c <Path of the tempest configuration file to use> -v -- tempest.api.identity.admin.v2.test_roles ./refstack-client test -c <Path of the tempest configuration file to use> -v -- --regex tempest.api.identity.admin.v2.test_roles
or :: or ::
./refstack-client test -c <Path of the tempest configuration file to use> -v -- tempest.api.identity.v2.test_token ./refstack-client test -c <Path of the tempest configuration file to use> -v -- --regex tempest.api.identity.v2.test_token
5. Run tests. 5. Run tests.
@ -79,10 +83,10 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
e. Adding the ``-r`` option with a string will prefix the JSON result file with the e. Adding the ``-r`` option with a string will prefix the JSON result file with the
given string (e.g. ``-r my-test`` will yield a result file like given string (e.g. ``-r my-test`` will yield a result file like
'my-test-0.json'). 'my-test-0.json').
f. Adding ``--`` enables you to pass arbitary arguments to the Tempest runner. f. Adding ``--`` enables you to pass arbitrary arguments to the ostestr runner.
After the first ``--``, all other subsequent arguments will be passed to After the first ``--``, all other subsequent arguments will be passed to
the Tempest runner as is. This is mainly used for quick verification of the the ostestr runner as is. This is mainly used for quick verification of the
target test cases. (e.g. ``-- tempest.api.identity.v2.test_token``) target test cases. (e.g. ``-- --regex tempest.api.identity.v2.test_token``)
Use ``./refstack-client test --help`` for the full list of arguments. Use ``./refstack-client test --help`` for the full list of arguments.
@ -129,8 +133,8 @@ configuration, you can activate a working Tempest environment by
switching to that directory and using the installed dependencies. switching to that directory and using the installed dependencies.
1. ``cd .tempest`` 1. ``cd .tempest``
2. run tempest with ``./run_tempest.sh -V`` or ``source ./.venv/bin/activate`` 2. ``source ./.venv/bin/activate``
and run tests manually with ``testr``. and run tests manually with ``ostestr``.
This will make the entire Tempest environment available for you to run, This will make the entire Tempest environment available for you to run,
including the ``run_tempest`` script and ``testr``. including the ``ostestr`` and ``testr`` commands.

15
refstack_client/list_parser.py
View File

@ -205,3 +205,18 @@ class TestListParser(object):
base_test_ids) base_test_ids)
list_file = self._write_normalized_test_list(full_capability_test_ids) list_file = self._write_normalized_test_list(full_capability_test_ids)
return list_file return list_file
def create_whitelist(self, list_location):
"""This takes in a test list file, get normalized, and get whitelist
test IDs.
Ex:
'tempest.test1[id-2,gate]' -> id-2
'tempest.test2[id-3,smoke](scenario)' -> id-3
:param list_location: file path or URL location of list file
"""
normalized_list = open(self.get_normalized_test_list(list_location),
'r').read()
# Keep the IDs
test_ids = re.sub("[\,\]].*", "", re.sub(".*.\[", "", normalized_list))
return self._write_normalized_test_list(test_ids.split('\n'))

64
refstack_client/refstack_client.py
View File

@ -73,7 +73,9 @@ class RefstackClient:
self.logger.addHandler(self.console_log_handle) self.logger.addHandler(self.console_log_handle)
self.args = args self.args = args
self.tempest_dir = '.tempest' self.current_dir = os.path.dirname(os.path.realpath(__file__))
self.refstack_dir = os.path.dirname(self.current_dir)
self.tempest_dir = os.path.join(self.refstack_dir, '.tempest')
# set default log level to INFO. # set default log level to INFO.
if self.args.silent: if self.args.silent:
@ -102,14 +104,9 @@ class RefstackClient:
"does not exist: %s" % self.tempest_dir) "does not exist: %s" % self.tempest_dir)
exit(1) exit(1)
self.tempest_script = os.path.join(self.tempest_dir,
'run_tempest.sh')
self.conf_file = self.args.conf_file self.conf_file = self.args.conf_file
self.conf = ConfigParser.SafeConfigParser() self.conf = ConfigParser.SafeConfigParser()
self.conf.read(self.args.conf_file) self.conf.read(self.args.conf_file)
self.tempest_script = os.path.join(self.tempest_dir,
'run_tempest.sh')
def _prep_upload(self): def _prep_upload(self):
'''Prepare an upload to the RefStack_api''' '''Prepare an upload to the RefStack_api'''
@ -406,28 +403,32 @@ class RefstackClient:
self.logger.info("Starting Tempest test...") self.logger.info("Starting Tempest test...")
start_time = time.time() start_time = time.time()
# Run the tempest script, specifying the conf file, the flag # Run the ostestr command, conf file specified at _prep_test method
# telling it to use a virtual environment (-V), and the flag # Use virtual environment (wrapper script)
# telling it to run the tests serially (-t). # telling it to run the tests serially (--serial).
cmd = [self.tempest_script, '-C', self.conf_file, '-V', '-t'] wrapper = os.path.join(self.tempest_dir, 'tools', 'with_venv.sh')
cmd = [wrapper, 'ostestr', '--serial', '--no-slowest']
# If a test list was specified, have it take precedence. # If a test list was specified, have it take precedence.
if self.args.test_list: if self.args.test_list:
self.logger.info("Normalizing test list...") self.logger.info("Normalizing test list...")
parser = TestListParser(os.path.abspath(self.tempest_dir)) parser = TestListParser(os.path.abspath(self.tempest_dir))
parser.setup_venv(self.logger.getEffectiveLevel()) parser.setup_venv(self.logger.getEffectiveLevel())
list_file = parser.get_normalized_test_list(self.args.test_list) # get whitelist
list_file = parser.create_whitelist(self.args.test_list)
if list_file: if list_file:
cmd += ('--', '--load-list', list_file) cmd += ('--whitelist_file', list_file)
else: else:
self.logger.error("Error normalizing passed in test list.") self.logger.error("Error normalizing passed in test list.")
exit(1) exit(1)
elif 'arbitrary_args' in self.args: elif 'arbitrary_args' in self.args:
# Add the tempest test cases to test as arguments. If no test # Additional arguments for ostestr runner
# cases are specified, then all Tempest API tests will be run. # otherwise run all Tempest API tests.
cmd += self.args.arbitrary_args # keep usage(-- testCaseName)
tmp = self.args.arbitrary_args[1:]
if tmp:
cmd += (tmp if tmp[0].startswith('-') else ['--regex'] + tmp)
else: else:
cmd += ['--', "tempest.api"] cmd += ['--regex', "tempest.api"]
# If there were two verbose flags, show tempest results. # If there were two verbose flags, show tempest results.
if self.args.verbose > 0: if self.args.verbose > 0:
@ -437,12 +438,14 @@ class RefstackClient:
# results to stderr. # results to stderr.
stderr = open(os.devnull, 'w') stderr = open(os.devnull, 'w')
# Execute the tempest test script in a subprocess. # Execute the ostestr command in a subprocess.
os.chdir(self.tempest_dir)
process = subprocess.Popen(cmd, stderr=stderr) process = subprocess.Popen(cmd, stderr=stderr)
process.communicate() process.communicate()
os.chdir(self.refstack_dir)
# If the subunit file was created, then the Tempest test was at least # If the subunit file was created, then test cases were executed via
# started successfully. # ostestr and there is test output to process.
if os.path.isfile(results_file): if os.path.isfile(results_file):
end_time = time.time() end_time = time.time()
elapsed = end_time - start_time elapsed = end_time - start_time
@ -472,8 +475,17 @@ class RefstackClient:
self.post_results(self.args.url, content, self.post_results(self.args.url, content,
sign_with=self.args.priv_key) sign_with=self.args.priv_key)
else: else:
self.logger.error("Problem executing Tempest script. Exit code %d", msg1 = ("ostestr command did not generate a results file under "
process.returncode) "the Refstack .tempest/.testrepository directory."
"Review command and try again.")
msg2 = ("Problem executing ostestr command. Results file not "
"generated hence no file to upload. "
"Review arbitrary arguments.")
if process.returncode != 0:
self.logger.warning(msg1)
if self.args.upload:
self.logger.error(msg2)
return process.returncode return process.returncode
def upload(self): def upload(self):
@ -704,12 +716,12 @@ def parse_cli_args(args=None):
parser_test.add_argument('arbitrary_args', parser_test.add_argument('arbitrary_args',
nargs=argparse.REMAINDER, nargs=argparse.REMAINDER,
help='After the first "--", you can pass ' help='After the first "--", you can pass '
'arbitrary arguments to the Tempest runner. ' 'arbitrary arguments to the ostestr runner. '
'This can be used for running specific test ' 'This can be used for running specific test '
'cases or test lists. Some examples are: ' 'cases or test lists. Some examples are: '
'-- tempest.api.compute.images.test_list_' '-- --regex tempest.api.compute.images.'
'image_filters ' 'test_list_image_filters OR '
'-- --load-list /tmp/test-list.txt') '-- --whitelist_file /tmp/testid-list.txt')
parser_test.set_defaults(func="test") parser_test.set_defaults(func="test")
# List command # List command

34
refstack_client/tests/unit/test_client.py
View File

@ -569,8 +569,8 @@ class TestRefstackClient(unittest.TestCase):
client.test() client.test()
mock_popen.assert_called_with( mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'-V', '-t', '--', 'tempest.api.compute'], '--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None stderr=None
) )
@ -600,8 +600,8 @@ class TestRefstackClient(unittest.TestCase):
client._get_cpid_from_keystone = MagicMock() client._get_cpid_from_keystone = MagicMock()
client.test() client.test()
mock_popen.assert_called_with( mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'-V', '-t', '--', 'tempest.api.compute'], '--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None stderr=None
) )
@ -633,8 +633,8 @@ class TestRefstackClient(unittest.TestCase):
return_value='test-id') return_value='test-id')
client.test() client.test()
mock_popen.assert_called_with( mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'-V', '-t', '--', 'tempest.api.compute'], '--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None stderr=None
) )
@ -664,15 +664,16 @@ class TestRefstackClient(unittest.TestCase):
client.post_results = MagicMock() client.post_results = MagicMock()
lp.TestListParser.get_normalized_test_list = MagicMock( lp.TestListParser.get_normalized_test_list = MagicMock(
return_value="/tmp/some-list") return_value="/tmp/some-list")
lp.TestListParser.create_whitelist = MagicMock(
return_value="/tmp/some-list")
client._get_keystone_config = MagicMock( client._get_keystone_config = MagicMock(
return_value=self.v2_config) return_value=self.v2_config)
client.test() client.test()
lp.TestListParser.get_normalized_test_list.assert_called_with( lp.TestListParser.create_whitelist.assert_called_with('test-list.txt')
'test-list.txt')
mock_popen.assert_called_with( mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ['%s/tools/with_venv.sh' % self.test_path, 'ostestr', '--serial',
'-V', '-t', '--', '--load-list', '/tmp/some-list'], '--no-slowest', '--whitelist_file', '/tmp/some-list'],
stderr=None stderr=None
) )
@ -719,15 +720,14 @@ class TestRefstackClient(unittest.TestCase):
client.test() client.test()
mock_popen.assert_called_with( mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name, ['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'-V', '-t', '--', 'tempest.api.compute'], '--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None stderr=None
) )
directory = os.path.dirname(os.path.realpath(__file__))
# Since '1' is in the next-stream file, we expect the JSON output file # Since '1' is in the next-stream file, we expect the JSON output file
# to be 'my-test-1.json'. # to be 'my-test-1.json'.
expected_file = directory + "/.testrepository/my-test-1.json" expected_file = os.path.join(self.test_path, '.testrepository',
'my-test-1.json')
client._save_json_results.assert_called_with(mock.ANY, expected_file) client._save_json_results.assert_called_with(mock.ANY, expected_file)
def test_failed_run(self): def test_failed_run(self):
@ -740,12 +740,12 @@ class TestRefstackClient(unittest.TestCase):
client = rc.RefstackClient(args) client = rc.RefstackClient(args)
client.tempest_dir = self.test_path client.tempest_dir = self.test_path
self.mock_data() self.mock_data()
client.logger.error = MagicMock() client.logger.warning = MagicMock()
client._get_keystone_config = MagicMock( client._get_keystone_config = MagicMock(
return_value=self.v2_config) return_value=self.v2_config)
client._get_cpid_from_keystone = MagicMock() client._get_cpid_from_keystone = MagicMock()
client.test() client.test()
self.assertTrue(client.logger.error.called) self.assertTrue(client.logger.warning.called)
def test_upload(self): def test_upload(self):
""" """