Add and require conf-file argument

* The user can now pass in a tempest configuration file to the
  refstack-client script. A tempest config file is now required.
* The tempest script was changed from run_tests.sh to run_tempest.sh
  as run_tests.sh is meant for the tempest unit tests.
* Made it so test results are printed out only with a verbosity count
  of two.
* Deprecated/broken code was removed.
* We now keep track of where the subunit data is saved.

Change-Id: I8a98f45c2f50368512d711a675a07dea7e23ad04

refstack-client/refstack-client

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 #
 # Copyright (c) 2014 Piston Cloud Computing, Inc. All Rights Reserved.
 #
@@ -16,39 +17,26 @@
 
 
 """
-Run tempest and upload results to Refstack.
+Run Tempest and upload results to Refstack.
 
-This module deals with generating a working tempest conf file from
+This module runs the Tempest test suite on an OpenStack environment given a
-the environment variables (usually populated by sourcing openrc.sh)
+Tempest configuration file.
-
-This is currently just built for Havana tempest.
-We're also just starting with nova-net and not neutron support.
-TODO:
-* generate config
 
 """
+
 import argparse
-import ConfigParser
 import logging
 import os
 import requests
 import subprocess
 
 
-class Test:
+class RefstackClient:
     log_format = "%(asctime)s %(name)s %(levelname)s %(message)s"
-    base_config_file = os.path.join(
-        os.path.abspath(os.path.dirname(__file__)),
-        'tempest.conf')
-
-    expected_env = {'OS_AUTH_URL': "uri",
-                    'OS_TENANT_NAME': "admin_tenant_name",
-                    'OS_USERNAME': "admin_username",
-                    'OS_PASSWORD': "admin_password"}
 
     def __init__(self, args):
         '''Prepare a tempest test against a cloud.'''
-        self.logger = logging.getLogger("execute_test")
+        self.logger = logging.getLogger("refstack_client")
         self.console_log_handle = logging.StreamHandler()
         self.console_log_handle.setFormatter(
             logging.Formatter(self.log_format))
@@ -59,86 +47,83 @@ class Test:
         elif args.verbose == 1:
             self.logger.setLevel(logging.INFO)
         else:
-            self.logger.setLevel(logging.CRITICAL)
+            self.logger.setLevel(logging.ERROR)
 
         # assign local vars to match args
         self.url = args.url
-        self.test_id = args.test_id
         self.tempest_dir = args.tempest_dir
-        self.output_conf = os.path.join(self.tempest_dir, 'tempest.config')
+        self.tempest_script = os.path.join(self.tempest_dir, 'run_tempest.sh')
-        self.tempest_script = os.path.join(self.tempest_dir, 'run_tests.sh')
+        self.test_cases = args.test_cases
+        self.verbose = args.verbose
 
-        # set up object config parser
+        # Check that the config file exists.
-        self.config_parser = ConfigParser.SafeConfigParser()
+        if not os.path.isfile(args.conf_file):
-        self.config_parser.read(self.base_config_file)
+            self.logger.error("File not valid: %s" % args.conf_file)
+            exit()
 
-        #import config
+        self.conf_file = args.conf_file
-        self.import_config_from_env()
+        self.results_file = self._get_subunit_output_file()
 
-        # write the config file
+    def post_results(self):
-        self.config_parser.write(open(self.output_conf, 'w+'))
-
-        #prep cloud based on new config
-        self.prepare_cloud()
-
-    def import_config_from_env(self):
-        """create config from environment variables if set otherwise
-        promt user for values"""
-        for env_var, key in self.expected_env.items():
-            if not os.environ.get(env_var):
-                value = raw_input("Enter value for %s: " % env_var)
-            else:
-                value = os.environ.get(env_var)
-
-            self.config_parser.set('identity', key, value)
-
-    def prepare_cloud(self):
-        """triggers tempest auto generate code to add users and setup image"""
-        #TODO davidlenwell: figure out how to import and run code in
-        #                   scripts/prep_cloud.py
-        pass
-
-    def post_test_result(self):
         '''Post the combined results back to the server.'''
-        self.logger.info('Send back the result')
 
-        with open(self.result, 'rb') as result_file:
+        # TODO(cdiep): This method needs to generate results in a format
-            # clean out any passwords or sensitive data.
+        # defined in https://review.openstack.org/#/c/105901/, and post those
-            result_file = self._scrub_sensitive_data(result_file)
+        # results using the v1 API.
-            # send the file home.
-            payload = {'file': result_file, 'test_id': self.test_id}
-            self._post_to_api('post-result', payload)
 
-    def _scrub_sensitive_data(self, results_file):
+        self.logger.info('Sending the Tempest test results to the Refstack '
-        '''stub function for cleaning the results before sending them back.'''
+                         'API server.')
-        return results_file
 
-    def _post_status(self, message):
+    def _get_subunit_output_file(self):
-        '''this function posts status back to the api server
+        '''This method reads from the next-stream file in the .testrepository
-        if it has a test_id to report it with.'''
+           directory of the given Tempest path. The integer here is the name
-        if self.test_id:
+           of the file where subunit output will be saved to.'''
-            payload = {'test_id': self.test_id, 'status': message}
-            self._post_to_api('update-test-status', payload)
-
-    def _post_to_api(self, method, payload):
-        '''This is a utility function for posting to the api. it is used by
-        both _post_status and post_test_result'''
         try:
-            url = '%s/v1/%s' % (self.url, method)
+            subunit_file = open(os.path.join(
-            requests.post(url, payload)
+                                self.tempest_dir, '.testrepository',
-            self.logger.info('posted successfully')
+                                'next-stream'), 'r').read()
-        except Exception as e:
+        except (IOError, OSError):
-            #TODO davidlenwell: be more explicit with exceptions.
+            self.logger.debug('The .testrepository/next-stream file was not '
-            self.logger.critical('failed to post %s - %s ' % (url,e))
+                              'found. Assuming subunit results will be stored '
-            raise
+                              'in file 0.')
+
+            # Testr saves the first test stream to .testrepository/0 when
+            # there is a newly generated .testrepository directory.
+            subunit_file = "0"
+
+        return os.path.join(self.tempest_dir, '.testrepository', subunit_file)
 
     def run(self):
         '''Execute tempest test against the cloud.'''
         try:
-            cmd = (self.tempest_script, '-C', self.output_conf)
+            self.logger.info("Starting Tempest test...")
-            subprocess.check_output(cmd, shell=True)
 
-            self.post_test_result()
+            # Run the tempest script, specifying the output_conf file and the
+            # flag telling it to not use a virtual environment.
+            cmd = (self.tempest_script, '-C', self.conf_file, '-N')
+
+            # Add the tempest test cases to test as arguments.
+            if self.test_cases:
+                cmd += ('--', self.test_cases)
+
+            # If there were two verbose flags, show tempest results.
+            if self.verbose > 1:
+                stderr = None
+            else:
+                # Suppress tempest results output. Note that testr prints
+                # results to stderr.
+                stderr = open(os.devnull, 'w')
+
+            # Execute the tempest test script in a subprocess.
+            process = subprocess.Popen(cmd, stderr=stderr)
+            process.communicate()
+
+            # TODO(cdiep): Enable post_test_result once the code is ready.
+            # self.post_results()
+
+            self.logger.info('Tempest test complete.')
+            self.logger.debug('Subunit results located in: %s' %
+                              self.results_file)
 
         except subprocess.CalledProcessError as e:
             self.logger.error('%s failed to complete' % (e))
@@ -150,39 +135,42 @@ if __name__ == '__main__':
                                      formatter_class=argparse.
                                      ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('-s', '--silent',
+    parser.add_argument('-v', '--verbose',
-                        action='store_true',
+                        action='count',
-                        help='rigged for silent running')
+                        help='Show verbose output. Note that -vv will show '
+                             'Tempest test result output.')
 
-    parser.add_argument("-v", "--verbose",
+    parser.add_argument('--url',
-                        action="count",
-                        help="show verbose output")
-
-    parser.add_argument("--url",
                         action='store',
                         required=False,
                         default='https://api.refstack.org',
                         type=str,
-                        help="refstack API url \
+                        help='Refstack API URL to post results to (e.g. --url '
-                               retrieve configurations. i.e.:\
+                             'https://127.0.0.1:8000).')
-                               --url https://127.0.0.1:8000")
 
-    parser.add_argument("--test-id",
+    parser.add_argument('--tempest-dir',
-                        action='store',
-                        required=False,
-                        dest='test_id',
-                        type=int,
-                        help="refstack test ID i.e.:\
-                                       --test-id 1234 ")
-
-    parser.add_argument("--tempest-dir",
                         action='store',
                         required=False,
                         dest='tempest_dir',
                         default='test_runner/src/tempest',
-                        help="tempest directory path")
+                        help='Path of the tempest project directory.')
+
+    parser.add_argument('-c', '--conf-file',
+                        action='store',
+                        required=True,
+                        dest='conf_file',
+                        type=str,
+                        help='Path of the tempest configuration file to use.')
+
+    parser.add_argument('-t', '--test-cases',
+                        action='store',
+                        required=False,
+                        dest='test_cases',
+                        type=str,
+                        help='Specify a subset of test cases to run '
+                             '(e.g. --test-cases tempest.api.compute).')
 
     args = parser.parse_args()
 
-    test = Test(args)
+    test = RefstackClient(args)
-    test.run
+    test.run()