Fixed Test Coverage Handling

- Coverage command was causing tests to
  stop after first run (OptParse was exiting
  so had to override the behaviour)
- Also added logging, error handling, and
  combining ot coverage files between runs.
- Output summary report at end of run

Change-Id: Ibec98009b16226de9a68da3be3d7b258d6d771a4

keystone/test/__init__.py

@@ -418,7 +418,16 @@ class KeystoneTest(object):
         # run the keystone server
         logger.info("Starting the keystone server...")
 
-        parser = optparse.OptionParser(version='%%prog %s' %
+        class SilentOptParser(optparse.OptionParser):
+            """ Class used to prevent OptionParser from exiting when it detects
+            options coming in for nose/testing """
+            def exit():
+                pass
+
+            def error(self, msg):
+                pass
+
+        parser = SilentOptParser(version='%%prog %s' %
                                        keystone.version.version())
         common_group = config.add_common_options(parser)
         config.add_log_options(parser)

keystone/test/etc/sql_no_hpidm.conf.template

@@ -27,13 +27,13 @@ backend_entities = ['Endpoints', 'Credentials',  'EndpointTemplates', 'Tenant',
 
 [pipeline:admin]
 pipeline =
-        urlrewritefilter
+        urlnormalizer
         d5_compat
         admin_api
 
 [pipeline:keystone-legacy-auth]
 pipeline =
-        urlrewritefilter
+        urlnormalizer
         legacy_auth
         d5_compat
         service_api
@@ -44,8 +44,8 @@ paste.app_factory = keystone.server:service_app_factory
 [app:admin_api]
 paste.app_factory = keystone.server:admin_app_factory
 
-[filter:urlrewritefilter]
-paste.filter_factory = keystone.middleware.url:filter_factory
+[filter:urlnormalizer]
+paste.filter_factory = keystone.frontends.normalizer:filter_factory
 
 [filter:d5_compat]
 paste.filter_factory = keystone.frontends.d5_compat:filter_factory

run_tests.py

@@ -57,9 +57,10 @@ if __name__ == '__main__':
         TESTS = [t for t in TESTS if filter in str(t)]
         if not TESTS:
             print 'No test configuration by the name %s found' % filter
-            exit()
+            sys.exit(2)
     #Run test suites
     if len(TESTS) > 1:
+        directory = os.getcwd()
         for test_num, test_cls in enumerate(TESTS):
             # We've had problems with resetting SQLAlchemy, so we can fire off
             # a separate process for each test suite to guarantee the
@@ -82,19 +83,25 @@ if __name__ == '__main__':
                 except Exception, e:
                     print "Error:", e
                     logger.exception(e)
-                    sys.exit(2)
+                    sys.exit(1)
             # Collect coverage from each run. They'll be combined later in .sh
             if '--with-coverage' in sys.argv:
-                coverage_file = '.coverage.%s' % test_num
+                coverage_file = os.path.join(directory, ".coverage")
+                target_file = "%s.%s" % (coverage_file, test_cls.__name__)
                 try:
+                    if os.path.exists(target_file):
+                        logger.info("deleting %s" % target_file)
+                        os.unlink(target_file)
                     if os.path.exists(coverage_file):
-                        os.unlink(coverage_file)
-                    os.rename('.coverage', coverage_file)
+                        logger.info("Saving %s to %s" % (coverage_file,
+                                                         target_file))
+                        os.rename(coverage_file, target_file)
                 except Exception, e:
                     logger.exception(e)
-                    print "Failed to move .coverage file to %s: %s" % \
-                        (coverage_file, e)
-
+                    print ("Failed to move coverage file while running test"
+                           ": %s. Error reported was: %s" %
+                           (test_cls.__name__, e))
+                    sys.exit(1)
     else:
         for test_num, test_cls in enumerate(TESTS):
             try:
@@ -106,4 +113,4 @@ if __name__ == '__main__':
             except Exception, e:
                 print "Error:", e
                 logger.exception(e)
-                sys.exit(2)
+                sys.exit(1)

run_tests.sh

@@ -24,7 +24,7 @@ function usage {
   echo "  -p, --pep8               Just run pep8"
   echo "  -l, --pylint             Just run pylint"
   echo "  -j, --json               Just validate JSON"
-  echo "  -c, --coverage           Generate coverage report"
+  echo "  -c, --with-coverage      Generate coverage report"
   echo "  -h, --help               Print this usage message"
   echo "  --hide-elapsed           Don't print the elapsed time for each test along with slow test list"
   echo "  --verbose                Print additional logging"
@@ -57,7 +57,7 @@ function process_option {
       -p|--pep8) just_pep8=1;;
       -l|--pylint) just_pylint=1;;
       -j|--json) just_json=1;;
-      -c|--coverage) coverage=1;;
+      -c|--with-coverage) coverage=1;;
       -*) addlopts="$addlopts $1";;
       *) addlargs="$addlargs $1"
     esac
@@ -197,5 +197,6 @@ if [ $coverage -eq 1 ]; then
     echo "Generating coverage report in covhtml/"
     ${wrapper} coverage combine
     ${wrapper} coverage html -d covhtml -i
+    ${wrapper} coverage report --omit='/usr*,keystone/test*,.,setup.py,*egg*,/Library*,*.xml,*.tpl'
 fi