Add ability to upload test results

Split refstack-client into two subcommands: upload and test.
The 'test' command will run Tempest like before, while 'upload'
takes in a result file argument and will upload it to the Refstack
API url. Testing with refstack-client now saves a JSON result file
containing the content of what will or would be posted to the
Refstack API.

Change-Id: I28bcb75b51b77872f39e144ae8ffa7e64b26b233

README.rst

@@ -5,13 +5,13 @@ refstack-client is a command line utility that allows you to execute Tempest
 test runs based on configurations you specify.  When finished running Tempest
 it sends the passed test data back to the Refstack API server.
 
-**Usage**
+**Setup**
 
 We've created an "easy button" for Ubuntu, Centos, RHEL and openSuSe.
 
-$ ./setup_env.sh
+$ ./setup_env
 
-**Start testing**
+**Usage**
 
 1. Prepare a tempest configuration file that is customized to your cloud
    environment.
@@ -21,10 +21,9 @@ $ ./setup_env.sh
 
    source .venv/bin/activate
 
-4. Execute test by typing:
+4. Test your cloud by typing:
 
-   ./refstack-client -c "Path of the tempest configuration file\
+   ./refstack-client test -c <Path of the tempest configuration file to use>
-   to use"
 
    **Note:**
 
@@ -33,5 +32,21 @@ $ ./setup_env.sh
    c. Adding -t option will only test a particular test case or a test group.
       This option can be used for quick verification of the target test cases
       (i.e. -t "tempest.api.identity.admin.test_roles").
-   d. Adding --url option will upload the test results to a Refstack API server
+   d. Adding --url option will upload the test results to the specified
-      instead of the default Refstack API server.
+      Refstack API server instead of the default Refstack API server.
+      server instead of the default Refstack API server.
+   e. Adding --offline option will have your test results not be uploaded.
+
+
+   **Upload:**
+
+   If you previously ran a test with refstack-client using the --offline
+   option, you can upload your results to a Refstack API server by using the
+   following command:
+
+   ./refstack-client upload <Path of results file>
+
+   The results file is a JSON file generated by refstack-client when a test has
+   completed. This is saved in .venv/src/tempest/.testrepository. When you use
+   the 'upload' command, you can also override the Refstack API server
+   uploaded to with the --url option.

refstack-client

@@ -28,4 +28,4 @@ from refstack_client import refstack_client
 if __name__ == '__main__':
     args = refstack_client.parse_cli_args()
     test = refstack_client.RefstackClient(args)
-    test.run()
+    getattr(test, args.func)()

refstack_client/refstack_client.py

@@ -48,16 +48,17 @@ class RefstackClient:
             logging.Formatter(self.log_format))
         self.logger.addHandler(self.console_log_handle)
 
-        if args.verbose > 1:
+        self.args = args
+
+        if self.args.verbose > 1:
             self.logger.setLevel(logging.DEBUG)
-        elif args.verbose == 1:
+        elif self.args.verbose == 1:
             self.logger.setLevel(logging.INFO)
         else:
             self.logger.setLevel(logging.ERROR)
 
-        self.args = args
+    def _prep_test(self):
-        self.tempest_script = os.path.join(self.args.tempest_dir,
+        '''Prepare a tempest test against a cloud.'''
-                                           'run_tempest.sh')
 
         # Check that the config file exists.
         if not os.path.isfile(self.args.conf_file):
@@ -76,6 +77,16 @@ class RefstackClient:
         self.conf_file = self.args.conf_file
         self.conf = ConfigParser.SafeConfigParser()
         self.conf.read(self.args.conf_file)
+        self.tempest_script = os.path.join(self.args.tempest_dir,
+                                           'run_tempest.sh')
+
+    def _prep_upload(self):
+        '''Prepare an upload to the Refstack_api'''
+        if not os.path.isfile(self.args.file):
+            self.logger.error("File not valid: %s" % self.args.file)
+            exit(1)
+        self.logger.setLevel(logging.DEBUG)
+        self.upload_file = self.args.file
 
     def _get_next_stream_subunit_output_file(self, tempest_dir):
         '''This method reads from the next-stream file in the .testrepository
@@ -131,6 +142,12 @@ class RefstackClient:
         content['results'] = results
         return content
 
+    def _save_json_results(self, results, path):
+        '''Save the output results from the Tempest run as a JSON file'''
+        file = open(path, "w+")
+        file.write(json.dumps(results, indent=4, separators=(',', ': ')))
+        file.close()
+
     def get_passed_tests(self, result_file):
         '''Get a list of tests IDs that passed Tempest from a subunit file.'''
         subunit_processor = SubunitProcessor(result_file)
@@ -145,8 +162,9 @@ class RefstackClient:
         json_content = json.dumps(content)
         self.logger.debug('API request content: %s ' % json_content)
 
-    def run(self):
+    def test(self):
-        '''Execute tempest test against the cloud.'''
+        '''Execute Tempest test against the cloud.'''
+        self._prep_test()
         results_file = self._get_next_stream_subunit_output_file(
             self.args.tempest_dir)
         cpid = self._get_cpid_from_keystone(self.conf)
@@ -189,6 +207,11 @@ class RefstackClient:
             results = self.get_passed_tests(results_file)
             self.logger.info("Number of passed tests: %d" % len(results))
 
+            content = self._form_result_content(cpid, duration, results)
+            json_path = results_file + ".json"
+            self._save_json_results(content, json_path)
+            self.logger.info('JSON results saved in: %s' % json_path)
+
             # If the user did not specify the offline argument, then upload
             # the results.
             if not self.args.offline:
@@ -198,50 +221,82 @@ class RefstackClient:
             self.logger.error("Problem executing Tempest script. Exit code %d",
                               process.returncode)
 
+    def upload(self):
+        '''Perform upload to Refstack URL.'''
+        self._prep_upload()
+        json_file = open(self.upload_file)
+        json_data = json.load(json_file)
+        json_file.close()
+        self.post_results(self.args.url, json_data)
+
 
 def parse_cli_args(args=None):
 
-    parser = argparse.ArgumentParser(description='Starts a tempest test',
+    usage_string = ('refstack-client [-h] {upload,test} ...\n\n'
+                    'To see help on specific argument, do:\n'
+                    'refstack-client <ARG> -h')
+
+    parser = argparse.ArgumentParser(description='Refstack-client arguments',
                                      formatter_class=argparse.
-                                     ArgumentDefaultsHelpFormatter)
+                                     ArgumentDefaultsHelpFormatter,
+                                     usage=usage_string)
 
-    parser.add_argument('-v', '--verbose',
+    subparsers = parser.add_subparsers(help='Available subcommands.')
+
+    # Arguments that go with all subcommands.
+    shared_args = argparse.ArgumentParser(add_help=False)
+    shared_args.add_argument('-v', '--verbose',
                              action='count',
-                        help='Show verbose output. Note that -vv will show '
+                             help='Show verbose output.')
-                             'Tempest test result output.')
 
-    parser.add_argument('--offline',
+    shared_args.add_argument('--url',
-                        action='store_true',
-                        help='Do not upload test results after running '
-                             'Tempest.')
-
-    parser.add_argument('--url',
                              action='store',
                              required=False,
                              default='https://api.refstack.org',
                              type=str,
-                        help='Refstack API URL to post results to (e.g. --url '
+                             help='Refstack API URL to upload results to '
-                             'https://127.0.0.1:8000).')
+                                  '(--url https://127.0.0.1:8000).')
 
-    parser.add_argument('--tempest-dir',
+    # Upload command
+    parser_upload = subparsers.add_parser('upload', parents=[shared_args],
+                                          help='Upload an existing result '
+                                               'file. ')
+    parser_upload.add_argument('file',
+                               type=str,
+                               help='Path of JSON results file.')
+    parser_upload.set_defaults(func="upload")
+
+    # Test command
+    parser_test = subparsers.add_parser('test', parents=[shared_args],
+                                        help='Run Tempest against a cloud.')
+
+    parser_test.add_argument('--tempest-dir',
                              action='store',
                              required=False,
                              dest='tempest_dir',
                              default='.venv/src/tempest',
-                        help='Path of the tempest project directory.')
+                             help='Path of the Tempest project directory.')
 
-    parser.add_argument('-c', '--conf-file',
+    parser_test.add_argument('-c', '--conf-file',
                              action='store',
                              required=True,
                              dest='conf_file',
                              type=str,
-                        help='Path of the tempest configuration file to use.')
+                             help='Path of the Tempest configuration file to '
+                                  'use.')
 
-    parser.add_argument('-t', '--test-cases',
+    parser_test.add_argument('-t', '--test-cases',
                              action='store',
                              required=False,
                              dest='test_cases',
                              type=str,
                              help='Specify a subset of test cases to run '
                                   '(e.g. --test-cases tempest.api.compute).')
+
+    parser_test.add_argument('--offline',
+                             action='store_true',
+                             help='Do not upload test results after running '
+                                  'Tempest.')
+    parser_test.set_defaults(func="test")
+
     return parser.parse_args(args=args)

refstack_client/tests/unit/.testrepository/0.json

@@ -0,0 +1,8 @@
+{
+    "duration_seconds": 0,
+    "cpid": "test-id",
+    "results": [
+        "tempest.passed.test"
+    ]
+}
+

refstack_client/tests/unit/test_client.py

@@ -15,7 +15,9 @@
 #
 
 import logging
+import json
 import os
+import tempfile
 
 import mock
 from mock import MagicMock
@@ -51,7 +53,8 @@ class TestRefstackClient(unittest.TestCase):
             conf_file_name = self.conf_file_name
         if tempest_dir is None:
             tempest_dir = self.test_path
-        argv = ['-c', conf_file_name,
+        argv = ['test',
+                '-c', conf_file_name,
                 '--tempest-dir', tempest_dir,
                 '--test-cases', 'tempest.api.compute',
                 '--url', '0.0.0.0']
@@ -86,14 +89,17 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client._prep_test()
         self.assertEqual(client.logger.level, logging.ERROR)
 
         args = rc.parse_cli_args(self.mock_argv(verbose='-v'))
         client = rc.RefstackClient(args)
+        client._prep_test()
         self.assertEqual(client.logger.level, logging.INFO)
 
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
+        client._prep_test()
         self.assertEqual(client.logger.level, logging.DEBUG)
 
     def test_get_next_stream_subunit_output_file(self):
@@ -128,6 +134,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client._prep_test()
         self.mock_keystone()
         cpid = client._get_cpid_from_keystone(client.conf)
         self.ks_client_builder.assert_called_with(
@@ -142,6 +149,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv())
         client = rc.RefstackClient(args)
+        client._prep_test()
         client.conf.remove_option('identity', 'admin_tenant_id')
         client.conf.set('identity', 'admin_tenant_name', 'admin_tenant_name')
         self.mock_keystone()
@@ -159,6 +167,7 @@ class TestRefstackClient(unittest.TestCase):
         """
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
+        client._prep_test()
         client.conf.remove_option('identity', 'admin_tenant_id')
         self.assertRaises(SystemExit, client._get_cpid_from_keystone,
                           client.conf)
@@ -175,6 +184,25 @@ class TestRefstackClient(unittest.TestCase):
                     'results': ['tempest.sample.test']}
         self.assertEqual(expected, content)
 
+    def test_save_json_result(self):
+        """
+        Test that the results are properly written to a JSON file.
+        """
+        args = rc.parse_cli_args(self.mock_argv())
+        client = rc.RefstackClient(args)
+        results = {'cpid': 1,
+                   'duration_seconds': 1,
+                   'results': ['tempest.sample.test']}
+        temp_file = tempfile.NamedTemporaryFile()
+        client._save_json_results(results, temp_file.name)
+
+        # Get the JSON that was written to the file and make sure it
+        # matches the expected value.
+        json_file = open(temp_file.name)
+        json_data = json.load(json_file)
+        json_file.close()
+        self.assertEqual(results, json_data)
+
     def test_get_passed_tests(self):
         """
         Test that only passing tests are retrieved from a subunit file.
@@ -201,7 +229,7 @@ class TestRefstackClient(unittest.TestCase):
         client.get_passed_tests = MagicMock(return_value=['test'])
         client.post_results = MagicMock()
         client._save_json_results = MagicMock()
-        client.run()
+        client.test()
         mock_popen.assert_called_with(
             ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
              '-N', '-t', '--', 'tempest.api.compute'),
@@ -229,7 +257,7 @@ class TestRefstackClient(unittest.TestCase):
         client.get_passed_tests = MagicMock(return_value=['test'])
         client.post_results = MagicMock()
         client._save_json_results = MagicMock()
-        client.run()
+        client.test()
         mock_popen.assert_called_with(
             ('%s/run_tempest.sh' % self.test_path, '-C', self.conf_file_name,
              '-N', '-t', '--', 'tempest.api.compute'),
@@ -245,14 +273,16 @@ class TestRefstackClient(unittest.TestCase):
         Test when a nonexistent configuration file is passed in.
         """
         args = rc.parse_cli_args(self.mock_argv(conf_file_name='ptn-khl'))
-        self.assertRaises(SystemExit, rc.RefstackClient, args)
+        client = rc.RefstackClient(args)
+        self.assertRaises(SystemExit, client.test)
 
     def test_run_tempest_nonexisting_directory(self):
         """
         Test when a nonexistent Tempest directory is passed in.
         """
         args = rc.parse_cli_args(self.mock_argv(tempest_dir='/does/not/exist'))
-        self.assertRaises(SystemExit, rc.RefstackClient, args)
+        client = rc.RefstackClient(args)
+        self.assertRaises(SystemExit, client.test)
 
     def test_failed_run(self):
         """
@@ -264,5 +294,33 @@ class TestRefstackClient(unittest.TestCase):
         args = rc.parse_cli_args(self.mock_argv(verbose='-vv'))
         client = rc.RefstackClient(args)
         client.logger.error = MagicMock()
-        client.run()
+        client.test()
         self.assertTrue(client.logger.error.called)
+
+    def test_upload(self):
+        """
+        Test that the upload command runs as expected.
+        """
+        upload_file_path = self.test_path + "/.testrepository/0.json"
+        args = rc.parse_cli_args(['upload', upload_file_path,
+                                  '--url', 'http://api.test.org'])
+        client = rc.RefstackClient(args)
+
+        client.post_results = MagicMock()
+        client.upload()
+        expected_json = {'duration_seconds': 0,
+                         'cpid': 'test-id',
+                         'results': ['tempest.passed.test']}
+
+        client.post_results.assert_called_with('http://api.test.org',
+                                               expected_json)
+
+    def test_upload_nonexisting_file(self):
+        """
+        Test that the upload file does not exist
+        """
+        upload_file_path = self.test_path + "/.testrepository/foo.json"
+        args = rc.parse_cli_args(['upload', upload_file_path,
+                                  '--url', 'http://api.test.org'])
+        client = rc.RefstackClient(args)
+        self.assertRaises(SystemExit, client.upload)