openinfra/refstack-client
Code Issues Proposed changes

Add ability to upload test results

Browse Source 
Split refstack-client into two subcommands: upload and test.
The 'test' command will run Tempest like before, while 'upload'
takes in a result file argument and will upload it to the Refstack
API url. Testing with refstack-client now saves a JSON result file
containing the content of what will or would be posted to the
Refstack API.

Change-Id: I28bcb75b51b77872f39e144ae8ffa7e64b26b233
This commit is contained in:
Paul Van Eck 2014-10-14 16:44:12 -07:00
parent 6e6ced6436
commit 6c3f3a1a02
5 changed files with 194 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
README.rst
View File

@ -5,13 +5,13 @@ refstack-client is a command line utility that allows you to execute Tempest
test runs based on configurations you specify. When finished running Tempest
it sends the passed test data back to the Refstack API server.
**Usage**
**Setup**
We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
$ ./setup_env.sh
$ ./setup_env
**Start testing**
**Usage**
1. Prepare a tempest configuration file that is customized to your cloud
environment.
@ -21,10 +21,9 @@ $ ./setup_env.sh
source .venv/bin/activate
4. Execute test by typing:
4. Test your cloud by typing:
./refstack-client -c "Path of the tempest configuration file\
to use"
./refstack-client test -c <Path of the tempest configuration file to use>
**Note:**
@ -33,5 +32,21 @@ $ ./setup_env.sh
c. Adding -t option will only test a particular test case or a test group.
This option can be used for quick verification of the target test cases
(i.e. -t "tempest.api.identity.admin.test_roles").
d. Adding --url option will upload the test results to a Refstack API server
instead of the default Refstack API server.
d. Adding --url option will upload the test results to the specified
Refstack API server instead of the default Refstack API server.
server instead of the default Refstack API server.
e. Adding --offline option will have your test results not be uploaded.
**Upload:**
If you previously ran a test with refstack-client using the --offline
option, you can upload your results to a Refstack API server by using the
following command:
./refstack-client upload <Path of results file>
The results file is a JSON file generated by refstack-client when a test has
completed. This is saved in .venv/src/tempest/.testrepository. When you use
the 'upload' command, you can also override the Refstack API server
uploaded to with the --url option.

2
refstack-client
View File

@ -28,4 +28,4 @@ from refstack_client import refstack_client
if __name__ == '__main__':
args = refstack_client.parse_cli_args()
test = refstack_client.RefstackClient(args)
test.run()
getattr(test, args.func)()

141
refstack_client/refstack_client.py
View File

@ -48,16 +48,17 @@ class RefstackClient:
logging.Formatter(self.log_format))
self.logger.addHandler(self.console_log_handle)
if args.verbose > 1:
self.args = args
if self.args.verbose > 1:
self.logger.setLevel(logging.DEBUG)
elif args.verbose == 1:
elif self.args.verbose == 1:
self.logger.setLevel(logging.INFO)
else:
self.logger.setLevel(logging.ERROR)
self.args = args
self.tempest_script = os.path.join(self.args.tempest_dir,
'run_tempest.sh')
def _prep_test(self):
'''Prepare a tempest test against a cloud.'''
# Check that the config file exists.
if not os.path.isfile(self.args.conf_file):
@ -76,6 +77,16 @@ class RefstackClient:
self.conf_file = self.args.conf_file
self.conf = ConfigParser.SafeConfigParser()
self.conf.read(self.args.conf_file)
self.tempest_script = os.path.join(self.args.tempest_dir,
'run_tempest.sh')
def _prep_upload(self):
'''Prepare an upload to the Refstack_api'''
if not os.path.isfile(self.args.file):
self.logger.error("File not valid: %s" % self.args.file)
exit(1)
self.logger.setLevel(logging.DEBUG)
self.upload_file = self.args.file
def _get_next_stream_subunit_output_file(self, tempest_dir):
'''This method reads from the next-stream file in the .testrepository
@ -131,6 +142,12 @@ class RefstackClient:
content['results'] = results
return content
def _save_json_results(self, results, path):
'''Save the output results from the Tempest run as a JSON file'''
file = open(path, "w+")
file.write(json.dumps(results, indent=4, separators=(',', ': ')))
file.close()
def get_passed_tests(self, result_file):
'''Get a list of tests IDs that passed Tempest from a subunit file.'''
subunit_processor = SubunitProcessor(result_file)
@ -145,8 +162,9 @@ class RefstackClient:
json_content = json.dumps(content)
self.logger.debug('API request content: %s ' % json_content)
def run(self):
'''Execute tempest test against the cloud.'''
def test(self):
'''Execute Tempest test against the cloud.'''
self._prep_test()
results_file = self._get_next_stream_subunit_output_file(
self.args.tempest_dir)
cpid = self._get_cpid_from_keystone(self.conf)
@ -189,6 +207,11 @@ class RefstackClient:
results = self.get_passed_tests(results_file)
self.logger.info("Number of passed tests: %d" % len(results))
content = self._form_result_content(cpid, duration, results)
json_path = results_file + ".json"
self._save_json_results(content, json_path)
self.logger.info('JSON results saved in: %s' % json_path)
# If the user did not specify the offline argument, then upload
# the results.
if not self.args.offline:
@ -198,50 +221,82 @@ class RefstackClient:
self.logger.error("Problem executing Tempest script. Exit code %d",
process.returncode)
def upload(self):
'''Perform upload to Refstack URL.'''
self._prep_upload()
json_file = open(self.upload_file)
json_data = json.load(json_file)
json_file.close()
self.post_results(self.args.url, json_data)
def parse_cli_args(args=None):
parser = argparse.ArgumentParser(description='Starts a tempest test',
usage_string = ('refstack-client [-h] {upload,test} ...\n\n'
'To see help on specific argument, do:\n'
'refstack-client <ARG> -h')
parser = argparse.ArgumentParser(description='Refstack-client arguments',
formatter_class=argparse.
ArgumentDefaultsHelpFormatter)
ArgumentDefaultsHelpFormatter,
usage=usage_string)
parser.add_argument('-v', '--verbose',
action='count',
help='Show verbose output. Note that -vv will show '
'Tempest test result output.')
subparsers = parser.add_subparsers(help='Available subcommands.')
parser.add_argument('--offline',
action='store_true',
help='Do not upload test results after running '
'Tempest.')
# Arguments that go with all subcommands.
shared_args = argparse.ArgumentParser(add_help=False)
shared_args.add_argument('-v', '--verbose',
action='count',
help='Show verbose output.')
parser.add_argument('--url',
action='store',
required=False,
default='https://api.refstack.org',
type=str,
help='Refstack API URL to post results to (e.g. --url '
'https://127.0.0.1:8000).')
shared_args.add_argument('--url',
action='store',
required=False,
default='https://api.refstack.org',
type=str,
help='Refstack API URL to upload results to '
'(--url https://127.0.0.1:8000).')
parser.add_argument('--tempest-dir',
action='store',
required=False,
dest='tempest_dir',
default='.venv/src/tempest',
help='Path of the tempest project directory.')
# Upload command
parser_upload = subparsers.add_parser('upload', parents=[shared_args],
help='Upload an existing result '
'file. ')
parser_upload.add_argument('file',
type=str,
help='Path of JSON results file.')
parser_upload.set_defaults(func="upload")
parser.add_argument('-c', '--conf-file',
action='store',
required=True,
dest='conf_file',
type=str,
help='Path of the tempest configuration file to use.')
# Test command
parser_test = subparsers.add_parser('test', parents=[shared_args],
help='Run Tempest against a cloud.')
parser_test.add_argument('--tempest-dir',
action='store',
required=False,
dest='tempest_dir',
default='.venv/src/tempest',
help='Path of the Tempest project directory.')
parser_test.add_argument('-c', '--conf-file',
action='store',
required=True,
dest='conf_file',
type=str,
help='Path of the Tempest configuration file to '
'use.')
parser_test.add_argument('-t', '--test-cases',
action='store',
required=False,
dest='test_cases',
type=str,
help='Specify a subset of test cases to run '
'(e.g. --test-cases tempest.api.compute).')
parser_test.add_argument('--offline',
action='store_true',
help='Do not upload test results after running '
'Tempest.')
parser_test.set_defaults(func="test")
parser.add_argument('-t', '--test-cases',
action='store',
required=False,
dest='test_cases',
type=str,
help='Specify a subset of test cases to run '
'(e.g. --test-cases tempest.api.compute).')
return parser.parse_args(args=args)

8
refstack_client/tests/unit/.testrepository/0.json Normal file
View File

@ -0,0 +1,8 @@
{
"duration_seconds": 0,
"cpid": "test-id",
"results": [
"tempest.passed.test"
]
}

70
refstack_client/tests/unit/test_client.py
View File

@ -15,7 +15,9 @@
#
import logging
import json
import os
import tempfile
import mock
from mock import MagicMock
@ -51,7 +53,8 @@ class TestRefstackClient(unittest.TestCase):
conf_file_name = self.conf_file_name
if tempest_dir is None:
tempest_dir = self.test_path
argv = ['-c', conf_file_name,
argv = ['test',
'-c', conf_file_name,
'--tempest-dir', tempest_dir,
'--test-cases', 'tempest.api.compute',
'--url', '0.0.0.0']
@ -86,14 +89,17 @@ class TestRefstackClient(unittest.TestCase):
"""
args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args)
client._prep_test()
self.assertEqual(client.logger.level, logging.ERROR)
args = rc.parse_cli_args(self.mock_argv(verbose='-v'))
client = rc.RefstackClient(args)
client._prep_test()
self.assertEqual(client.logger.level, logging.INFO)
args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
client = rc.RefstackClient(args)
client._prep_test()
self.assertEqual(client.logger.level, logging.DEBUG)
def test_get_next_stream_subunit_output_file(self):
@ -128,6 +134,7 @@ class TestRefstackClient(unittest.TestCase):
"""
args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args)
client._prep_test()
self.mock_keystone()
cpid = client._get_cpid_from_keystone(client.conf)
self.ks_client_builder.assert_called_with(
@ -142,6 +149,7 @@ class TestRefstackClient(unittest.TestCase):
"""
args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args)
client._prep_test()
client.conf.remove_option('identity', 'admin_tenant_id')
client.conf.set('identity', 'admin_tenant_name', 'admin_tenant_name')
self.mock_keystone()
@ -159,6 +167,7 @@ class TestRefstackClient(unittest.TestCase):
"""
args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
client = rc.RefstackClient(args)
client._prep_test()
client.conf.remove_option('identity', 'admin_tenant_id')
self.assertRaises(SystemExit, client._get_cpid_from_keystone,
client.conf)
@ -175,6 +184,25 @@ class TestRefstackClient(unittest.TestCase):
'results': ['tempest.sample.test']}
self.assertEqual(expected, content)
def test_save_json_result(self):
"""
Test that the results are properly written to a JSON file.
"""
args = rc.parse_cli_args(self.mock_argv())
client = rc.RefstackClient(args)
results = {'cpid': 1,
'duration_seconds': 1,
'results': ['tempest.sample.test']}
temp_file = tempfile.NamedTemporaryFile()
client._save_json_results(results, temp_file.name)
# Get the JSON that was written to the file and make sure it
# matches the expected value.
json_file = open(temp_file.name)
json_data = json.load(json_file)
json_file.close()
self.assertEqual(results, json_data)
def test_get_passed_tests(self):
"""
Test that only passing tests are retrieved from a subunit file.
@ -201,7 +229,7 @@ class TestRefstackClient(unittest.TestCase):
client.get_passed_tests = MagicMock(return_value=['test'])
client.post_results = MagicMock()
client._save_json_results = MagicMock()
client.run()
client.test()
mock_popen.assert_called_with(
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-N', '-t', '--', 'tempest.api.compute'),
@ -229,7 +257,7 @@ class TestRefstackClient(unittest.TestCase):
client.get_passed_tests = MagicMock(return_value=['test'])
client.post_results = MagicMock()
client._save_json_results = MagicMock()
client.run()
client.test()
mock_popen.assert_called_with(
('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
'-N', '-t', '--', 'tempest.api.compute'),
@ -245,14 +273,16 @@ class TestRefstackClient(unittest.TestCase):
Test when a nonexistent configuration file is passed in.
"""
args = rc.parse_cli_args(self.mock_argv(conf_file_name='ptn-khl'))
self.assertRaises(SystemExit, rc.RefstackClient, args)
client = rc.RefstackClient(args)
self.assertRaises(SystemExit, client.test)
def test_run_tempest_nonexisting_directory(self):
"""
Test when a nonexistent Tempest directory is passed in.
"""
args = rc.parse_cli_args(self.mock_argv(tempest_dir='/does/not/exist'))
self.assertRaises(SystemExit, rc.RefstackClient, args)
client = rc.RefstackClient(args)
self.assertRaises(SystemExit, client.test)
def test_failed_run(self):
"""
@ -264,5 +294,33 @@ class TestRefstackClient(unittest.TestCase):
args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
client = rc.RefstackClient(args)
client.logger.error = MagicMock()
client.run()
client.test()
self.assertTrue(client.logger.error.called)
def test_upload(self):
"""
Test that the upload command runs as expected.
"""
upload_file_path = self.test_path + "/.testrepository/0.json"
args = rc.parse_cli_args(['upload', upload_file_path,
'--url', 'http://api.test.org'])
client = rc.RefstackClient(args)
client.post_results = MagicMock()
client.upload()
expected_json = {'duration_seconds': 0,
'cpid': 'test-id',
'results': ['tempest.passed.test']}
client.post_results.assert_called_with('http://api.test.org',
expected_json)
def test_upload_nonexisting_file(self):
"""
Test that the upload file does not exist
"""
upload_file_path = self.test_path + "/.testrepository/foo.json"
args = rc.parse_cli_args(['upload', upload_file_path,
'--url', 'http://api.test.org'])
client = rc.RefstackClient(args)
self.assertRaises(SystemExit, client.upload)