Use testr to run tests

This is taken largely from python-novaclient and Heat

Change-Id: I8f2eb9185ac073e9ee97bb9c757a4c3c2f8d6d6b

.gitignore

@@ -1,5 +1,7 @@
 .coverage
 .venv
+.testrepository
+subunit.log
 *,cover
 cover
 *.pyc

.testr.conf

@@ -0,0 +1,4 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

run_tests.sh

@@ -1,49 +1,169 @@
 #!/bin/bash
 
+set -eu
+
 function usage {
   echo "Usage: $0 [OPTION]..."
-  echo "Run python-ceilometerclient's test suite(s)"
+  echo "Run python-ceilometerclient test suite"
   echo ""
+  echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
+  echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -s, --no-site-packages   Isolate the virtualenv from the global Python environment"
+  echo "  -x, --stop               Stop running tests after the first error or failure."
+  echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
   echo "  -p, --pep8               Just run pep8"
+  echo "  -P, --no-pep8            Don't run pep8"
+  echo "  -c, --coverage           Generate coverage report"
   echo "  -h, --help               Print this usage message"
+  echo "  --hide-elapsed           Don't print the elapsed time for each test along with slow test list"
   echo ""
-  echo "This script is deprecated and currently retained for compatibility."
+  echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
-  echo 'You can run the full test suite for multiple environments by running "tox".'
+  echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
-  echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only'
+  echo "      prefer to run tests NOT in a virtual environment, simply pass the -N option."
-  echo 'the pep8 tests with "tox -e pep8".'
   exit
 }
 
-command -v tox > /dev/null 2>&1
-if [ $? -ne 0 ]; then
-  echo 'This script requires "tox" to run.'
-  echo 'You can install it with "pip install tox".'
-  exit 1; 
-fi
-
-just_pep8=0
-
 function process_option {
   case "$1" in
     -h|--help) usage;;
-    -p|--pep8) let just_pep8=1;;
+    -V|--virtual-env) always_venv=1; never_venv=0;;
+    -N|--no-virtual-env) always_venv=0; never_venv=1;;
+    -s|--no-site-packages) no_site_packages=1;;
+    -f|--force) force=1;;
+    -p|--pep8) just_pep8=1;;
+    -P|--no-pep8) no_pep8=1;;
+    -c|--coverage) coverage=1;;
+    -*) testropts="$testropts $1";;
+    *) testrargs="$testrargs $1"
   esac
 }
 
+venv=.venv
+with_venv=tools/with_venv.sh
+always_venv=0
+never_venv=0
+force=0
+no_site_packages=0
+installvenvopts=
+testrargs=
+testropts=
+wrapper=""
+just_pep8=0
+no_pep8=0
+coverage=0
+
+LANG=en_US.UTF-8
+LANGUAGE=en_US:en
+LC_ALL=C
+
 for arg in "$@"; do
   process_option $arg
 done
 
+if [ $no_site_packages -eq 1 ]; then
+  installvenvopts="--no-site-packages"
+fi
+
+function init_testr {
+  if [ ! -d .testrepository ]; then
+    ${wrapper} testr init
+  fi
+}
+
+function run_tests {
+  # Cleanup *pyc
+  ${wrapper} find . -type f -name "*.pyc" -delete
+
+  if [ $coverage -eq 1 ]; then
+    # Do not test test_coverage_ext when gathering coverage.
+    if [ "x$testrargs" = "x" ]; then
+      testrargs="^(?!.*test_coverage_ext).*$"
+    fi
+    export PYTHON="${wrapper} coverage run --source novaclient --parallel-mode"
+  fi
+  # Just run the test suites in current environment
+  set +e
+  TESTRTESTS="$TESTRTESTS $testrargs"
+  echo "Running \`${wrapper} $TESTRTESTS\`"
+  ${wrapper} $TESTRTESTS
+  RESULT=$?
+  set -e
+
+  copy_subunit_log
+
+  return $RESULT
+}
+
+function copy_subunit_log {
+  LOGNAME=`cat .testrepository/next-stream`
+  LOGNAME=$(($LOGNAME - 1))
+  LOGNAME=".testrepository/${LOGNAME}"
+  cp $LOGNAME subunit.log
+}
+
+function run_pep8 {
+  echo "Running pep8 ..."
+  srcfiles="--exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*"
+  srcfiles+=",*egg,build ."
+  # Just run PEP8 in current environment
+  #
+  ignore="--ignore=E12,E711,E721,E712"
+  ${wrapper} pep8 ${ignore} --show-source ${srcfiles}
+}
+
+TESTRTESTS="testr run --parallel $testropts"
+
+if [ $never_venv -eq 0 ]
+then
+  # Remove the virtual environment if --force used
+  if [ $force -eq 1 ]; then
+    echo "Cleaning virtualenv..."
+    rm -rf ${venv}
+  fi
+  if [ -e ${venv} ]; then
+    wrapper="${with_venv}"
+  else
+    if [ $always_venv -eq 1 ]; then
+      # Automatically install the virtualenv
+      python tools/install_venv.py $installvenvopts
+      wrapper="${with_venv}"
+    else
+      echo -e "No virtual environment found...create one? (Y/n) \c"
+      read use_ve
+      if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
+        # Install the virtualenv and run the test suite in it
+        python tools/install_venv.py $installvenvopts
+        wrapper=${with_venv}
+      fi
+    fi
+  fi
+fi
+
+# Delete old coverage data from previous runs
+if [ $coverage -eq 1 ]; then
+    ${wrapper} coverage erase
+fi
+
 if [ $just_pep8 -eq 1 ]; then
-  tox -e pep8
+    run_pep8
     exit
 fi
 
-tox -e py27 $toxargs 2>&1 | tee run_tests.err.log  || exit
+init_testr
-if [ ${PIPESTATUS[0]} -ne 0 ]; then
+run_tests
-  exit ${PIPESTATUS[0]}
+
+# NOTE(sirp): we only want to run pep8 when we're running the full-test suite,
+# not when we're running tests individually. To handle this, we need to
+# distinguish between options (noseopts), which begin with a '-', and
+# arguments (testrargs).
+if [ -z "$testrargs" ]; then
+  if [ $no_pep8 -eq 0 ]; then
+    run_pep8
+  fi
 fi
 
-if [ -z "$toxargs" ]; then
+if [ $coverage -eq 1 ]; then
-  tox -e pep8
+    echo "Generating coverage report in covhtml/"
+    ${wrapper} coverage combine
+    ${wrapper} coverage html --include='novaclient/*' --omit='novaclient/openstack/common/*' -d covhtml -i
 fi

setup.cfg

@@ -1,11 +1,3 @@
-[nosetests]
-cover-package = ceilometerclient
-cover-html = true
-cover-erase = true
-cover-inclusive = true
-verbosity=2
-detailed-errors=1
-
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build

setup.py

@@ -24,16 +24,15 @@ def read(fname):
 setuptools.setup(
     name=project,
     version=setup.get_version(project),
-    author='Ceilometer Developers',
+    author='OpenStack',
     author_email='openstack-dev@lists.openstack.org',
     description="Client library for ceilometer",
     long_description=read('README.md'),
-    license='Apache',
+    license="Apache License, Version 2.0",
     url='https://github.com/openstack/python-ceilometerclient',
     packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
     include_package_data=True,
     install_requires=setup.parse_requirements(),
-    test_suite="nose.collector",
     cmdclass=setup.get_cmdclass(),
     classifiers=[
         'Development Status :: 4 - Beta',

tests/test_shell.py

@@ -1,9 +1,11 @@
 import cStringIO
-import os
 import httplib2
+import re
 import sys
 
-import mox
+import fixtures
+from testtools import matchers
+
 try:
     import json
 except ImportError:
@@ -15,50 +17,26 @@ from ceilometerclient.v1 import client as v1client
 import ceilometerclient.shell
 from tests import utils
 
-
+FAKE_ENV = {'OS_USERNAME': 'username',
-class ShellValidationTest(utils.BaseTestCase):
+            'OS_PASSWORD': 'password',
-
+            'OS_TENANT_NAME': 'tenant_name',
-    def shell_error(self, argstr, error_match):
+            'OS_AUTH_URL': 'http://no.where'}
-        orig = sys.stderr
-        try:
-            sys.stderr = cStringIO.StringIO()
-            _shell = ceilometerclient.shell.CeilometerShell()
-            _shell.main(argstr.split())
-        except exc.CommandError as e:
-            self.assertRegexpMatches(e.__str__(), error_match)
-        else:
-            self.fail('Expected error matching: %s' % error_match)
-        finally:
-            err = sys.stderr.getvalue()
-            sys.stderr.close()
-            sys.stderr = orig
-        return err
 
 
 class ShellTest(utils.BaseTestCase):
+    re_options = re.DOTALL | re.MULTILINE
 
     # Patch os.environ to avoid required auth info.
+    def make_env(self, exclude=None):
+        env = dict((k, v) for k, v in FAKE_ENV.items() if k != exclude)
+        self.useFixture(fixtures.MonkeyPatch('os.environ', env))
+
     def setUp(self):
         super(ShellTest, self).setUp()
         self.m.StubOutWithMock(ksclient, 'Client')
         self.m.StubOutWithMock(v1client.Client, 'json_request')
         self.m.StubOutWithMock(v1client.Client, 'raw_request')
 
-        global _old_env
-        fake_env = {
-            'OS_USERNAME': 'username',
-            'OS_PASSWORD': 'password',
-            'OS_TENANT_NAME': 'tenant_name',
-            'OS_AUTH_URL': 'http://no.where',
-        }
-        _old_env, os.environ = os.environ, fake_env.copy()
-
-    def tearDown(self):
-        super(ShellTest, self).tearDown()
-        self.m.UnsetStubs()
-        global _old_env
-        os.environ = _old_env
-
     def shell(self, argstr):
         orig = sys.stdout
         try:
@@ -85,19 +63,21 @@ class ShellTest(utils.BaseTestCase):
 
     def test_help(self):
         required = [
-            '^usage: ceilometer',
+            '.*?^usage: ceilometer',
-            '(?m)^See "ceilometer help COMMAND" '
+            '.*?^See "ceilometer help COMMAND" '
             'for help on a specific command',
         ]
         for argstr in ['--help', 'help']:
             help_text = self.shell(argstr)
             for r in required:
-                self.assertRegexpMatches(help_text, r)
+                self.assertThat(help_text,
+                                matchers.MatchesRegex(r,
+                                                      self.re_options))
 
     def test_help_on_subcommand(self):
         required = [
-            '^usage: ceilometer meter-list',
+            '.*?^usage: ceilometer meter-list',
-            "(?m)^List the user's meter",
+            ".*?^List the user's meter",
         ]
         argstrings = [
             'help meter-list',
@@ -105,19 +85,9 @@ class ShellTest(utils.BaseTestCase):
         for argstr in argstrings:
             help_text = self.shell(argstr)
             for r in required:
-                self.assertRegexpMatches(help_text, r)
+                self.assertThat(help_text,
+                                matchers.MatchesRegex(r, self.re_options))
 
     def test_auth_param(self):
-        class TokenContext(object):
+        self.make_env(exclude='OS_USERNAME')
-            def __enter__(self):
-                fake_env = {
-                    'OS_AUTH_TOKEN': 'token',
-                    'CEILOMETER_URL': 'http://no.where'
-                }
-                self.old_env, os.environ = os.environ, fake_env.copy()
-
-            def __exit__(self, exc_type, exc_value, traceback):
-                os.environ = self.old_env
-
-        with TokenContext():
         self.test_help()

tests/utils.py

@@ -14,18 +14,21 @@
 #    under the License.
 
 import copy
+import fixtures
 import StringIO
 import mox
-import unittest2
+import testtools
 
 from ceilometerclient.common import http
 
 
-class BaseTestCase(unittest2.TestCase):
+class BaseTestCase(testtools.TestCase):
 
     def setUp(self):
         super(BaseTestCase, self).setUp()
         self.m = mox.Mox()
+        self.addCleanup(self.m.UnsetStubs)
+        self.useFixture(fixtures.FakeLogger())
 
 
 class FakeAPI(object):

tools/test-requires

@@ -1,11 +1,11 @@
 # This file is managed by openstack-depends
+coverage>=3.6
+discover
 distribute>=0.6.24
+fixtures>=0.3.12
 mox>=0.5.3
-nose
+pep8==1.4.5
-nose-exclude
+python-subunit
-nosehtmloutput>=0.0.3
-nosexcover
-openstack.nose_plugin>=0.7
-pep8==1.3.3
 sphinx>=1.1.2
-unittest2
+testrepository>=0.0.13
+testtools>=0.9.29

tox.ini

@@ -2,45 +2,28 @@
 envlist = py26,py27,pep8
 
 [testenv]
+sitepackages = True
 setenv = VIRTUAL_ENV={envdir}
-         NOSE_WITH_OPENSTACK=1
+         LANG=en_US.UTF-8
-         NOSE_OPENSTACK_COLOR=1
+         LANGUAGE=en_US:en
-         NOSE_OPENSTACK_RED=0.05
+         LC_ALL=C
-         NOSE_OPENSTACK_YELLOW=0.025
-         NOSE_OPENSTACK_SHOW_ELAPSED=1
 deps = -r{toxinidir}/tools/pip-requires
        -r{toxinidir}/tools/test-requires
-commands = nosetests
+commands =
+  python setup.py testr --testr-args='{posargs}'
+
+[tox:jenkins]
+sitepackages = True
+downloadcache = ~/cache/pip
 
 [testenv:pep8]
-deps = pep8==1.3.3
+deps = pep8==1.4.5
-commands = pep8 --repeat --show-source ceilometerclient setup.py
+commands =
+  pep8 --ignore=E12,E711,E721,E712 --show-source \
+    --exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build .
+
+[testenv:cover]
+commands = python setup.py testr --coverage --testr-args='{posargs}'
 
 [testenv:venv]
 commands = {posargs}
-
-[testenv:cover]
-commands = nosetests --cover-erase --cover-package=ceilometerclient --with-xcoverage
-
-[tox:jenkins]
-downloadcache = ~/cache/pip
-
-[testenv:jenkins26]
-basepython = python2.6
-setenv = NOSE_WITH_XUNIT=1
-deps = file://{toxinidir}/.cache.bundle
-
-[testenv:jenkins27]
-basepython = python2.7
-setenv = NOSE_WITH_XUNIT=1
-deps = file://{toxinidir}/.cache.bundle
-
-[testenv:jenkinscover]
-deps = file://{toxinidir}/.cache.bundle
-setenv = NOSE_WITH_XUNIT=1
-commands = nosetests --cover-erase --cover-package=ceilometerclient --with-xcoverage
-
-[testenv:jenkinsvenv]
-deps = file://{toxinidir}/.cache.bundle
-setenv = NOSE_WITH_XUNIT=1
-commands = {posargs}