Changes to testr as the test runner

run_tests.sh has also been deprecated.

Fixes-Bug: #1177924
Change-Id: I15c1707eb6a62c74a0021a48b89ae15f03fcfea8

.gitignore

@@ -28,3 +28,4 @@ keystone/tests/tmp/
 .project
 .pydevproject
 keystone/locale/*/LC_MESSAGES/*.mo
+.testrepository/

.testr.conf

@@ -0,0 +1,12 @@
+[DEFAULT]
+test_command=${PYTHON:-python} -m subunit.run discover \
+                -t ./ ./keystone/tests \
+                $LISTOPT $IDOPTION
+
+test_id_option=--load-list $IDFILE
+test_list_option=--list
+
+# NOTE(dstanek): Ensures that Keystone test never run in parallel.
+#                Please remove once the issues have been worked out.
+#                Bug: #1240052
+test_run_concurrency=echo 1

keystone/tests/backend_sql_disk.conf

@@ -1,2 +1,2 @@
 [sql]
-connection = sqlite:///tmp/test.db
+connection = sqlite:///keystone/tests/tmp/test.db

keystone/tests/core.py

@@ -25,7 +25,6 @@ import time
 
 from lxml import etree
 import mox
-import nose.exc
 from paste import deploy
 import stubout
 import testtools
@@ -74,9 +73,9 @@ from keystone.openstack.common import policy as common_policy  # noqa
 
 
 LOG = logging.getLogger(__name__)
-ROOTDIR = os.path.dirname(os.path.abspath('..'))
+TESTSDIR = os.path.dirname(os.path.abspath(__file__))
+ROOTDIR = os.path.normpath(os.path.join(TESTSDIR, '..', '..'))
 VENDOR = os.path.join(ROOTDIR, 'vendor')
-TESTSDIR = os.path.join(ROOTDIR, 'keystone', 'tests')
 ETCDIR = os.path.join(ROOTDIR, 'etc')
 TMPDIR = os.path.join(TESTSDIR, 'tmp')
 
@@ -490,7 +489,7 @@ class TestCase(NoModule, testtools.TestCase):
     def assertEqualXML(self, a, b):
         """Parses two XML documents from strings and compares the results.
 
-        This provides easy-to-read failures from nose.
+        This provides easy-to-read failures.
 
         """
         parser = etree.XMLParser(remove_blank_text=True)
@@ -510,13 +509,12 @@ class TestCase(NoModule, testtools.TestCase):
         b = canonical_xml(b)
         self.assertEqual(a.split('\n'), b.split('\n'))
 
-    @staticmethod
-    def skip_if_no_ipv6():
+    def skip_if_no_ipv6(self):
         try:
             s = socket.socket(socket.AF_INET6)
         except socket.error as e:
             if e.errno == errno.EAFNOSUPPORT:
-                raise nose.exc.SkipTest("IPv6 is not enabled in the system")
+                raise self.skipTest("IPv6 is not enabled in the system")
             else:
                 raise
         else:

keystone/tests/test_ipv6.py

@@ -24,11 +24,9 @@ CONF = config.CONF
 
 
 class IPv6TestCase(tests.TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.skip_if_no_ipv6()
 
     def setUp(self):
+        self.skip_if_no_ipv6()
         super(IPv6TestCase, self).setUp()
         self.load_backends()
 

keystone/tests/test_overrides.conf

@@ -21,6 +21,6 @@ debug_cache_backend = True
 proxies = keystone.tests.test_cache.CacheIsolatingProxy
 
 [signing]
-certfile = ../../examples/pki/certs/signing_cert.pem
-keyfile = ../../examples/pki/private/signing_key.pem
-ca_certs = ../../examples/pki/certs/cacert.pem
+certfile = examples/pki/certs/signing_cert.pem
+keyfile = examples/pki/private/signing_key.pem
+ca_certs = examples/pki/certs/cacert.pem

keystone/tests/test_v3_auth.py

@@ -1071,7 +1071,7 @@ class TestTokenRevoking(test_v3.RestfulTestCase):
 class TestAuthExternalDisabled(test_v3.RestfulTestCase):
     def config_files(self):
         list = self._config_file_list[:]
-        list.append('auth_plugin_external_disabled.conf')
+        list.append(tests.testsdir('auth_plugin_external_disabled.conf'))
         return list
 
     def test_remote_user_disabled(self):
@@ -1093,7 +1093,7 @@ class TestAuthExternalDomain(test_v3.RestfulTestCase):
 
     def config_files(self):
         list = self._config_file_list[:]
-        list.append('auth_plugin_external_domain.conf')
+        list.append(tests.testsdir('auth_plugin_external_domain.conf'))
         return list
 
     def test_remote_user_with_realm(self):

openstack-common.conf

@@ -3,6 +3,7 @@
 # The list of modules to copy from openstack-common
 module=db
 module=db.sqlalchemy
+module=colorizer
 module=crypto
 module=importutils
 module=install_venv_common

run_tests.sh

@@ -32,9 +32,6 @@ function usage {
   echo "  -P, --no-pep8            Don't run flake8"
   echo "  -c, --coverage           Generate coverage report"
   echo "  -h, --help               Print this usage message"
-  echo "  -xintegration            Ignore all keystoneclient test cases (integration tests)"
-  echo "  --hide-elapsed           Don't print the elapsed time for each test along with slow test list"
-  echo "  --standard-threads       Don't do the eventlet threading monkeypatch."
   echo ""
   echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
   echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
@@ -55,12 +52,8 @@ function process_option {
     -8|--8) short_flake8=1;;
     -P|--no-pep8) no_flake8=1;;
     -c|--coverage) coverage=1;;
-    -xintegration) nokeystoneclient=1;;
-    --standard-threads)
-        export STANDARD_THREADS=1
-        ;;
-    -*) noseopts="$noseopts $1";;
-    *) noseargs="$noseargs $1"
+    -*) testropts="$testropts $1";;
+    *) testrargs="$testrargs $1"
   esac
 }
 
@@ -69,14 +62,13 @@ with_venv=tools/with_venv.sh
 always_venv=0
 never_venv=0
 force=0
-noseargs=
-noseopts="--with-openstack --openstack-color"
+testrargs=
+testropts=--subunit
 wrapper=""
 just_flake8=0
 short_flake8=0
 no_flake8=0
 coverage=0
-nokeystoneclient=0
 recreate_db=1
 update=0
 
@@ -84,14 +76,11 @@ for arg in "$@"; do
   process_option $arg
 done
 
+TESTRTESTS="python setup.py testr"
+
 # If enabled, tell nose to collect coverage data
 if [ $coverage -eq 1 ]; then
-    noseopts="$noseopts --with-coverage --cover-package=keystone"
-fi
-
-if [ $nokeystoneclient -eq 1 ]; then
-    # disable the integration tests
-    noseopts="$noseopts -I test_keystoneclient* -I _test_import_auth_token.py"
+    TESTRTESTS="$TESTRTESTS --coverage"
 fi
 
 function cleanup_test_db {
@@ -103,19 +92,11 @@ function cleanup_test_db {
 }
 
 function run_tests {
-  # Just run the test suites in current environment
-  ${wrapper} $NOSETESTS
-  # If we get some short import error right away, print the error log directly
-  RESULT=$?
-  if [ "$RESULT" -ne "0" ];
-  then
-    ERRSIZE=`wc -l run_tests.log | awk '{print \$1}'`
-    if [ "$ERRSIZE" -lt "40" ];
-    then
-        cat run_tests.log
-    fi
-  fi
-  return $RESULT
+  set -e
+  echo ${wrapper}
+  ${wrapper} $TESTRTESTS --testr-args="$testropts $testrargs" | \
+      ${wrapper} subunit-2to1 | \
+      ${wrapper} tools/colorizer.py
 }
 
 function run_flake8 {
@@ -125,14 +106,16 @@ function run_flake8 {
       FLAGS=''
   fi
 
-
   echo "Running flake8 ..."
   # Just run flake8 in current environment
   echo ${wrapper} flake8 $FLAGS | tee pep8.txt
   ${wrapper} flake8 $FLAGS | tee pep8.txt
 }
 
-NOSETESTS="nosetests $noseopts $noseargs"
+echo "This script is now deprecated.  Please use tox instead."
+echo "Checkout http://tox.readthedocs.org/en/latest/ for information on tox."
+echo "[press enter to continue]"
+read
 
 if [ $never_venv -eq 0 ]
 then
@@ -188,15 +171,10 @@ run_tests
 
 # NOTE(sirp): we only want to run flake8 when we're running the full-test
 # suite, not when we're running tests individually. To handle this, we need to
-# distinguish between options (noseopts), which begin with a '-', and arguments
-# (noseargs).
-if [ -z "$noseargs" ]; then
+# distinguish between options (testropts), which begin with a '-', and arguments
+# (testrargs).
+if [ -z "$testrargs" ]; then
   if [ $no_flake8 -eq 0 ]; then
     run_flake8
   fi
 fi
-
-if [ $coverage -eq 1 ]; then
-    echo "Generating coverage report in covhtml/"
-    ${wrapper} coverage html -d covhtml -i
-fi

setup.cfg

@@ -55,15 +55,3 @@ mapping_file = babel.cfg
 output_file = keystone/locale/keystone.pot
 copyright_holder = OpenStack Foundation
 msgid_bugs_address = https://bugs.launchpad.net/keystone
-
-[nosetests]
-# NOTE(jkoelker) To run the test suite under nose install the following
-#                coverage http://pypi.python.org/pypi/coverage
-#                tissue http://pypi.python.org/pypi/tissue (pep8 checker)
-#                openstack-nose https://github.com/jkoelker/openstack-nose
-verbosity=2
-detailed-errors=1
-cover-package = keystone
-cover-html = true
-cover-erase = true
-where=keystone/tests

test-requirements.txt

@@ -15,17 +15,17 @@ python-ldap==2.3.13
 coverage
 # mock object framework
 mox
-# for test discovery and console feedback
-nose
-nosexcover
-openstack.nose_plugin
-nosehtmloutput
 # required to build documentation
 Sphinx>=1.1.2
-testtools>=0.9.32
 # test wsgi apps without starting an http server
 webtest
 
+extras
+discover
+python-subunit
+testrepository>=0.0.17
+testtools>=0.9.32
+
 # for python-keystoneclient
 # keystoneclient <0.2.1
 httplib2

tools/colorizer.py

@@ -0,0 +1,333 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2013, Nebula, Inc.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# Colorizer Code is borrowed from Twisted:
+# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
+#
+#    Permission is hereby granted, free of charge, to any person obtaining
+#    a copy of this software and associated documentation files (the
+#    "Software"), to deal in the Software without restriction, including
+#    without limitation the rights to use, copy, modify, merge, publish,
+#    distribute, sublicense, and/or sell copies of the Software, and to
+#    permit persons to whom the Software is furnished to do so, subject to
+#    the following conditions:
+#
+#    The above copyright notice and this permission notice shall be
+#    included in all copies or substantial portions of the Software.
+#
+#    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+#    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+#    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+#    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Display a subunit stream through a colorized unittest test runner."""
+
+import heapq
+import subunit
+import sys
+import unittest
+
+import testtools
+
+
+class _AnsiColorizer(object):
+    """Colorizer allows callers to write text in a particular color.
+
+    A colorizer is an object that loosely wraps around a stream, allowing
+    callers to write text to the stream in a particular color.
+
+    Colorizer classes must implement C{supported()} and C{write(text, color)}.
+    """
+    _colors = dict(black=30, red=31, green=32, yellow=33,
+                   blue=34, magenta=35, cyan=36, white=37)
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    def supported(cls, stream=sys.stdout):
+        """Check is the current platform supports coloring terminal output.
+
+        A class method that returns True if the current platform supports
+        coloring terminal output using this method. Returns False otherwise.
+        """
+        if not stream.isatty():
+            return False  # auto color only on TTYs
+        try:
+            import curses
+        except ImportError:
+            return False
+        else:
+            try:
+                try:
+                    return curses.tigetnum("colors") > 2
+                except curses.error:
+                    curses.setupterm()
+                    return curses.tigetnum("colors") > 2
+            except Exception:
+                # guess false in case of error
+                return False
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        """Write the given text to the stream in the given color.
+
+        @param text: Text to be written to the stream.
+
+        @param color: A string label for a color. e.g. 'red', 'white'.
+        """
+        color = self._colors[color]
+        self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text))
+
+
+class _Win32Colorizer(object):
+    """See _AnsiColorizer docstring."""
+    def __init__(self, stream):
+        import win32console
+        red, green, blue, bold = (win32console.FOREGROUND_RED,
+                                  win32console.FOREGROUND_GREEN,
+                                  win32console.FOREGROUND_BLUE,
+                                  win32console.FOREGROUND_INTENSITY)
+        self.stream = stream
+        self.screenBuffer = win32console.GetStdHandle(
+            win32console.STD_OUT_HANDLE)
+        self._colors = {
+            'normal': red | green | blue,
+            'red': red | bold,
+            'green': green | bold,
+            'blue': blue | bold,
+            'yellow': red | green | bold,
+            'magenta': red | blue | bold,
+            'cyan': green | blue | bold,
+            'white': red | green | blue | bold,
+        }
+
+    def supported(cls, stream=sys.stdout):
+        try:
+            import win32console
+            screenBuffer = win32console.GetStdHandle(
+                win32console.STD_OUT_HANDLE)
+        except ImportError:
+            return False
+        import pywintypes
+        try:
+            screenBuffer.SetConsoleTextAttribute(
+                win32console.FOREGROUND_RED |
+                win32console.FOREGROUND_GREEN |
+                win32console.FOREGROUND_BLUE)
+        except pywintypes.error:
+            return False
+        else:
+            return True
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        color = self._colors[color]
+        self.screenBuffer.SetConsoleTextAttribute(color)
+        self.stream.write(text)
+        self.screenBuffer.SetConsoleTextAttribute(self._colors['normal'])
+
+
+class _NullColorizer(object):
+    """See _AnsiColorizer docstring."""
+    def __init__(self, stream):
+        self.stream = stream
+
+    def supported(cls, stream=sys.stdout):
+        return True
+    supported = classmethod(supported)
+
+    def write(self, text, color):
+        self.stream.write(text)
+
+
+def get_elapsed_time_color(elapsed_time):
+    if elapsed_time > 1.0:
+        return 'red'
+    elif elapsed_time > 0.25:
+        return 'yellow'
+    else:
+        return 'green'
+
+
+class OpenStackTestResult(testtools.TestResult):
+    def __init__(self, stream, descriptions, verbosity):
+        super(OpenStackTestResult, self).__init__()
+        self.stream = stream
+        self.showAll = verbosity > 1
+        self.num_slow_tests = 10
+        self.slow_tests = []  # this is a fixed-sized heap
+        self.colorizer = None
+        # NOTE(vish): reset stdout for the terminal check
+        stdout = sys.stdout
+        sys.stdout = sys.__stdout__
+        for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
+            if colorizer.supported():
+                self.colorizer = colorizer(self.stream)
+                break
+        sys.stdout = stdout
+        self.start_time = None
+        self.last_time = {}
+        self.results = {}
+        self.last_written = None
+
+    def _writeElapsedTime(self, elapsed):
+        color = get_elapsed_time_color(elapsed)
+        self.colorizer.write("  %.2f" % elapsed, color)
+
+    def _addResult(self, test, *args):
+        try:
+            name = test.id()
+        except AttributeError:
+            name = 'Unknown.unknown'
+        test_class, test_name = name.rsplit('.', 1)
+
+        elapsed = (self._now() - self.start_time).total_seconds()
+        item = (elapsed, test_class, test_name)
+        if len(self.slow_tests) >= self.num_slow_tests:
+            heapq.heappushpop(self.slow_tests, item)
+        else:
+            heapq.heappush(self.slow_tests, item)
+
+        self.results.setdefault(test_class, [])
+        self.results[test_class].append((test_name, elapsed) + args)
+        self.last_time[test_class] = self._now()
+        self.writeTests()
+
+    def _writeResult(self, test_name, elapsed, long_result, color,
+                     short_result, success):
+        if self.showAll:
+            self.stream.write('    %s' % str(test_name).ljust(66))
+            self.colorizer.write(long_result, color)
+            if success:
+                self._writeElapsedTime(elapsed)
+            self.stream.writeln()
+        else:
+            self.colorizer.write(short_result, color)
+
+    def addSuccess(self, test):
+        super(OpenStackTestResult, self).addSuccess(test)
+        self._addResult(test, 'OK', 'green', '.', True)
+
+    def addFailure(self, test, err):
+        if test.id() == 'process-returncode':
+            return
+        super(OpenStackTestResult, self).addFailure(test, err)
+        self._addResult(test, 'FAIL', 'red', 'F', False)
+
+    def addError(self, test, err):
+        super(OpenStackTestResult, self).addFailure(test, err)
+        self._addResult(test, 'ERROR', 'red', 'E', False)
+
+    def addSkip(self, test, reason=None, details=None):
+        super(OpenStackTestResult, self).addSkip(test, reason, details)
+        self._addResult(test, 'SKIP', 'blue', 'S', True)
+
+    def startTest(self, test):
+        self.start_time = self._now()
+        super(OpenStackTestResult, self).startTest(test)
+
+    def writeTestCase(self, cls):
+        if not self.results.get(cls):
+            return
+        if cls != self.last_written:
+            self.colorizer.write(cls, 'white')
+            self.stream.writeln()
+        for result in self.results[cls]:
+            self._writeResult(*result)
+        del self.results[cls]
+        self.stream.flush()
+        self.last_written = cls
+
+    def writeTests(self):
+        time = self.last_time.get(self.last_written, self._now())
+        if not self.last_written or (self._now() - time).total_seconds() > 2.0:
+            diff = 3.0
+            while diff > 2.0:
+                classes = self.results.keys()
+                oldest = min(classes, key=lambda x: self.last_time[x])
+                diff = (self._now() - self.last_time[oldest]).total_seconds()
+                self.writeTestCase(oldest)
+        else:
+            self.writeTestCase(self.last_written)
+
+    def done(self):
+        self.stopTestRun()
+
+    def stopTestRun(self):
+        for cls in list(self.results.iterkeys()):
+            self.writeTestCase(cls)
+        self.stream.writeln()
+        self.writeSlowTests()
+
+    def writeSlowTests(self):
+        # Pare out 'fast' tests
+        slow_tests = [item for item in self.slow_tests
+                      if get_elapsed_time_color(item[0]) != 'green']
+        if slow_tests:
+            slow_total_time = sum(item[0] for item in slow_tests)
+            slow = ("Slowest %i tests took %.2f secs:"
+                    % (len(slow_tests), slow_total_time))
+            self.colorizer.write(slow, 'yellow')
+            self.stream.writeln()
+            last_cls = None
+            # sort by name
+            for elapsed, cls, name in sorted(slow_tests,
+                                             key=lambda x: x[1] + x[2]):
+                if cls != last_cls:
+                    self.colorizer.write(cls, 'white')
+                    self.stream.writeln()
+                last_cls = cls
+                self.stream.write('    %s' % str(name).ljust(68))
+                self._writeElapsedTime(elapsed)
+                self.stream.writeln()
+
+    def printErrors(self):
+        if self.showAll:
+            self.stream.writeln()
+        self.printErrorList('ERROR', self.errors)
+        self.printErrorList('FAIL', self.failures)
+
+    def printErrorList(self, flavor, errors):
+        for test, err in errors:
+            self.colorizer.write("=" * 70, 'red')
+            self.stream.writeln()
+            self.colorizer.write(flavor, 'red')
+            self.stream.writeln(": %s" % test.id())
+            self.colorizer.write("-" * 70, 'red')
+            self.stream.writeln()
+            self.stream.writeln("%s" % err)
+
+
+test = subunit.ProtocolTestCase(sys.stdin, passthrough=None)
+
+if sys.version_info[0:2] <= (2, 6):
+    runner = unittest.TextTestRunner(verbosity=2)
+else:
+    runner = unittest.TextTestRunner(verbosity=2,
+                                     resultclass=OpenStackTestResult)
+
+if runner.run(test).wasSuccessful():
+    exit_code = 0
+else:
+    exit_code = 1
+sys.exit(exit_code)

tox.ini

@@ -5,18 +5,14 @@ envlist = py26,py27,py33,pep8
 
 [testenv]
 usedevelop = True
-install_command = pip install {opts} {packages}
+install_command = pip install -U {opts} {packages}
 setenv = VIRTUAL_ENV={envdir}
-         NOSE_WITH_OPENSTACK=1
-         NOSE_OPENSTACK_COLOR=1
-         NOSE_OPENSTACK_RED=0.05
-         NOSE_OPENSTACK_YELLOW=0.025
-         NOSE_OPENSTACK_SHOW_ELAPSED=1
-         NOSE_OPENSTACK_STDOUT=1
+         LANG=en_US.UTF-8
+         LANGUAGE=en_US:en
+         LC_ALL=C
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-commands = python tools/patch_tox_venv.py
-           nosetests {posargs}
+commands = python setup.py testr --testr-args='{posargs}'
 
 [testenv:pep8]
 commands = flake8
@@ -26,9 +22,7 @@ downloadcache = ~/cache/pip
 
 [testenv:cover]
 setenv = VIRTUAL_ENV={envdir}
-         NOSE_WITH_COVERAGE=1
-         NOSE_COVER_HTML=1
-         NOSE_COVER_HTML_DIR={toxinidir}/cover
+commands = python setup.py testr --testr-args='--coverage {posargs}'
 
 [testenv:venv]
 commands = {posargs}