openinfra/refstack-client
Code Issues Proposed changes

Replace run_tempest.sh script with ostestr command

Browse Source 
QA community default test runner is ostestr, and using run_tempest.sh
script is no longer recommended. BP replaces run_tempest.sh script
with direct ostestr call passing the appropriate parameters.

Implements blueprint: implement-ostestr-refstackclient

Change-Id: If4ffed70c8320abb4b4c7bfd1b84ca7892272665
This commit is contained in:
Luz Cazares 2016-06-14 14:28:01 -07:00
parent 648220f054
commit e610492537
4 changed files with 82 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
README.rst
View File

@ -30,6 +30,10 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
1. Prepare a tempest configuration file that is customized to your cloud
environment.
Note: Use Tempest Pre-Provisioned credentials_ to provide user test accounts. ::
.. _credentials: http://docs.openstack.org/developer/tempest/configuration.html#pre-provisioned-credentials
2. Go into the refstack-client directory::
cd ~/refstack-client
@ -40,11 +44,11 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
4. Validate your setup by running a short test::
./refstack-client test -c <Path of the tempest configuration file to use> -v -- tempest.api.identity.admin.v2.test_roles
./refstack-client test -c <Path of the tempest configuration file to use> -v -- --regex tempest.api.identity.admin.v2.test_roles
or ::
./refstack-client test -c <Path of the tempest configuration file to use> -v -- tempest.api.identity.v2.test_token
./refstack-client test -c <Path of the tempest configuration file to use> -v -- --regex tempest.api.identity.v2.test_token
5. Run tests.
@ -79,10 +83,10 @@ We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
e. Adding the ``-r`` option with a string will prefix the JSON result file with the
given string (e.g. ``-r my-test`` will yield a result file like
'my-test-0.json').
f. Adding ``--`` enables you to pass arbitary arguments to the Tempest runner.
f. Adding ``--`` enables you to pass arbitrary arguments to the ostestr runner.
After the first ``--``, all other subsequent arguments will be passed to
the Tempest runner as is. This is mainly used for quick verification of the
target test cases. (e.g. ``-- tempest.api.identity.v2.test_token``)
the ostestr runner as is. This is mainly used for quick verification of the
target test cases. (e.g. ``-- --regex tempest.api.identity.v2.test_token``)
Use ``./refstack-client test --help`` for the full list of arguments.
@ -129,8 +133,8 @@ configuration, you can activate a working Tempest environment by
switching to that directory and using the installed dependencies.
1. ``cd .tempest``
2. run tempest with ``./run_tempest.sh -V`` or ``source ./.venv/bin/activate``
and run tests manually with ``testr``.
2. ``source ./.venv/bin/activate``
and run tests manually with ``ostestr``.
This will make the entire Tempest environment available for you to run,
including the ``run_tempest`` script and ``testr``.
including the ``ostestr`` and ``testr`` commands.

15
refstack_client/list_parser.py
View File

@ -205,3 +205,18 @@ class TestListParser(object):
base_test_ids)
list_file = self._write_normalized_test_list(full_capability_test_ids)
return list_file
def create_whitelist(self, list_location):
"""This takes in a test list file, get normalized, and get whitelist
test IDs.
Ex:
'tempest.test1[id-2,gate]' -> id-2
'tempest.test2[id-3,smoke](scenario)' -> id-3
:param list_location: file path or URL location of list file
"""
normalized_list = open(self.get_normalized_test_list(list_location),
'r').read()
# Keep the IDs
test_ids = re.sub("[\,\]].*", "", re.sub(".*.\[", "", normalized_list))
return self._write_normalized_test_list(test_ids.split('\n'))

64
refstack_client/refstack_client.py
View File

@ -73,7 +73,9 @@ class RefstackClient:
self.logger.addHandler(self.console_log_handle)
self.args = args
self.tempest_dir = '.tempest'
self.current_dir = os.path.dirname(os.path.realpath(__file__))
self.refstack_dir = os.path.dirname(self.current_dir)
self.tempest_dir = os.path.join(self.refstack_dir, '.tempest')
# set default log level to INFO.
if self.args.silent:
@ -102,14 +104,9 @@ class RefstackClient:
"does not exist: %s" % self.tempest_dir)
exit(1)
self.tempest_script = os.path.join(self.tempest_dir,
'run_tempest.sh')
self.conf_file = self.args.conf_file
self.conf = ConfigParser.SafeConfigParser()
self.conf.read(self.args.conf_file)
self.tempest_script = os.path.join(self.tempest_dir,
'run_tempest.sh')
def _prep_upload(self):
'''Prepare an upload to the RefStack_api'''
@ -406,28 +403,32 @@ class RefstackClient:
self.logger.info("Starting Tempest test...")
start_time = time.time()
# Run the tempest script, specifying the conf file, the flag
# telling it to use a virtual environment (-V), and the flag
# telling it to run the tests serially (-t).
cmd = [self.tempest_script, '-C', self.conf_file, '-V', '-t']
# Run the ostestr command, conf file specified at _prep_test method
# Use virtual environment (wrapper script)
# telling it to run the tests serially (--serial).
wrapper = os.path.join(self.tempest_dir, 'tools', 'with_venv.sh')
cmd = [wrapper, 'ostestr', '--serial', '--no-slowest']
# If a test list was specified, have it take precedence.
if self.args.test_list:
self.logger.info("Normalizing test list...")
parser = TestListParser(os.path.abspath(self.tempest_dir))
parser.setup_venv(self.logger.getEffectiveLevel())
list_file = parser.get_normalized_test_list(self.args.test_list)
# get whitelist
list_file = parser.create_whitelist(self.args.test_list)
if list_file:
cmd += ('--', '--load-list', list_file)
cmd += ('--whitelist_file', list_file)
else:
self.logger.error("Error normalizing passed in test list.")
exit(1)
elif 'arbitrary_args' in self.args:
# Add the tempest test cases to test as arguments. If no test
# cases are specified, then all Tempest API tests will be run.
cmd += self.args.arbitrary_args
# Additional arguments for ostestr runner
# otherwise run all Tempest API tests.
# keep usage(-- testCaseName)
tmp = self.args.arbitrary_args[1:]
if tmp:
cmd += (tmp if tmp[0].startswith('-') else ['--regex'] + tmp)
else:
cmd += ['--', "tempest.api"]
cmd += ['--regex', "tempest.api"]
# If there were two verbose flags, show tempest results.
if self.args.verbose > 0:
@ -437,12 +438,14 @@ class RefstackClient:
# results to stderr.
stderr = open(os.devnull, 'w')
# Execute the tempest test script in a subprocess.
# Execute the ostestr command in a subprocess.
os.chdir(self.tempest_dir)
process = subprocess.Popen(cmd, stderr=stderr)
process.communicate()
os.chdir(self.refstack_dir)
# If the subunit file was created, then the Tempest test was at least
# started successfully.
# If the subunit file was created, then test cases were executed via
# ostestr and there is test output to process.
if os.path.isfile(results_file):
end_time = time.time()
elapsed = end_time - start_time
@ -472,8 +475,17 @@ class RefstackClient:
self.post_results(self.args.url, content,
sign_with=self.args.priv_key)
else:
self.logger.error("Problem executing Tempest script. Exit code %d",
process.returncode)
msg1 = ("ostestr command did not generate a results file under "
"the Refstack .tempest/.testrepository directory."
"Review command and try again.")
msg2 = ("Problem executing ostestr command. Results file not "
"generated hence no file to upload. "
"Review arbitrary arguments.")
if process.returncode != 0:
self.logger.warning(msg1)
if self.args.upload:
self.logger.error(msg2)
return process.returncode
def upload(self):
@ -704,12 +716,12 @@ def parse_cli_args(args=None):
parser_test.add_argument('arbitrary_args',
nargs=argparse.REMAINDER,
help='After the first "--", you can pass '
'arbitrary arguments to the Tempest runner. '
'arbitrary arguments to the ostestr runner. '
'This can be used for running specific test '
'cases or test lists. Some examples are: '
'-- tempest.api.compute.images.test_list_'
'image_filters '
'-- --load-list /tmp/test-list.txt')
'-- --regex tempest.api.compute.images.'
'test_list_image_filters OR '
'-- --whitelist_file /tmp/testid-list.txt')
parser_test.set_defaults(func="test")
# List command

34
refstack_client/tests/unit/test_client.py
View File

@ -569,8 +569,8 @@ class TestRefstackClient(unittest.TestCase):
client.test()
mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-V', '-t', '--', 'tempest.api.compute'],
['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None
)
@ -600,8 +600,8 @@ class TestRefstackClient(unittest.TestCase):
client._get_cpid_from_keystone = MagicMock()
client.test()
mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-V', '-t', '--', 'tempest.api.compute'],
['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None
)
@ -633,8 +633,8 @@ class TestRefstackClient(unittest.TestCase):
return_value='test-id')
client.test()
mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-V', '-t', '--', 'tempest.api.compute'],
['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None
)
@ -664,15 +664,16 @@ class TestRefstackClient(unittest.TestCase):
client.post_results = MagicMock()
lp.TestListParser.get_normalized_test_list = MagicMock(
return_value="/tmp/some-list")
lp.TestListParser.create_whitelist = MagicMock(
return_value="/tmp/some-list")
client._get_keystone_config = MagicMock(
return_value=self.v2_config)
client.test()
lp.TestListParser.get_normalized_test_list.assert_called_with(
'test-list.txt')
lp.TestListParser.create_whitelist.assert_called_with('test-list.txt')
mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-V', '-t', '--', '--load-list', '/tmp/some-list'],
['%s/tools/with_venv.sh' % self.test_path, 'ostestr', '--serial',
'--no-slowest', '--whitelist_file', '/tmp/some-list'],
stderr=None
)
@ -719,15 +720,14 @@ class TestRefstackClient(unittest.TestCase):
client.test()
mock_popen.assert_called_with(
['%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-V', '-t', '--', 'tempest.api.compute'],
['%s/tools/with_venv.sh' % self.test_path, 'ostestr',
'--serial', '--no-slowest', '--regex', 'tempest.api.compute'],
stderr=None
)
directory = os.path.dirname(os.path.realpath(__file__))
# Since '1' is in the next-stream file, we expect the JSON output file
# to be 'my-test-1.json'.
expected_file = directory + "/.testrepository/my-test-1.json"
expected_file = os.path.join(self.test_path, '.testrepository',
'my-test-1.json')
client._save_json_results.assert_called_with(mock.ANY, expected_file)
def test_failed_run(self):
@ -740,12 +740,12 @@ class TestRefstackClient(unittest.TestCase):
client = rc.RefstackClient(args)
client.tempest_dir = self.test_path
self.mock_data()
client.logger.error = MagicMock()
client.logger.warning = MagicMock()
client._get_keystone_config = MagicMock(
return_value=self.v2_config)
client._get_cpid_from_keystone = MagicMock()
client.test()
self.assertTrue(client.logger.error.called)
self.assertTrue(client.logger.warning.called)
def test_upload(self):
"""