Use testr to run tests

This is taken largely from python-novaclient and Heat Change-Id: I8f2eb9185ac073e9ee97bb9c757a4c3c2f8d6d6b
2013-05-10 21:39:34 +10:00
parent 44885c83b1
commit 7091db40a9
9 changed files with 203 additions and 130 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,7 @@
 .coverage
 .venv
+.testrepository
+subunit.log
 *,cover
 cover
 *.pyc
--- a/.testr.conf
+++ b/.testr.conf
@@ -0,0 +1,4 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,49 +1,169 @@
 #!/bin/bash

+set -eu
+
 function usage {
  echo "Usage: $0 [OPTION]..."
-  echo "Run python-ceilometerclient's test suite(s)"
+  echo "Run python-ceilometerclient test suite"
  echo ""
+  echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
+  echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
+  echo "  -s, --no-site-packages   Isolate the virtualenv from the global Python environment"
+  echo "  -x, --stop               Stop running tests after the first error or failure."
+  echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
  echo "  -p, --pep8               Just run pep8"
+  echo "  -P, --no-pep8            Don't run pep8"
+  echo "  -c, --coverage           Generate coverage report"
  echo "  -h, --help               Print this usage message"
+  echo "  --hide-elapsed           Don't print the elapsed time for each test along with slow test list"
  echo ""
-  echo "This script is deprecated and currently retained for compatibility."
-  echo 'You can run the full test suite for multiple environments by running "tox".'
-  echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only'
-  echo 'the pep8 tests with "tox -e pep8".'
+  echo "Note: with no options specified, the script will try to run the tests in a virtual environment,"
+  echo "      If no virtualenv is found, the script will ask if you would like to create one.  If you "
+  echo "      prefer to run tests NOT in a virtual environment, simply pass the -N option."
  exit
 }

-command -v tox > /dev/null 2>&1
-if [ $? -ne 0 ]; then
-  echo 'This script requires "tox" to run.'
-  echo 'You can install it with "pip install tox".'
-  exit 1; 
-fi
-
-just_pep8=0
-
 function process_option {
  case "$1" in
    -h|--help) usage;;
-    -p|--pep8) let just_pep8=1;;
+    -V|--virtual-env) always_venv=1; never_venv=0;;
+    -N|--no-virtual-env) always_venv=0; never_venv=1;;
+    -s|--no-site-packages) no_site_packages=1;;
+    -f|--force) force=1;;
+    -p|--pep8) just_pep8=1;;
+    -P|--no-pep8) no_pep8=1;;
+    -c|--coverage) coverage=1;;
+    -*) testropts="$testropts $1";;
+    *) testrargs="$testrargs $1"
  esac
 }

+venv=.venv
+with_venv=tools/with_venv.sh
+always_venv=0
+never_venv=0
+force=0
+no_site_packages=0
+installvenvopts=
+testrargs=
+testropts=
+wrapper=""
+just_pep8=0
+no_pep8=0
+coverage=0
+
+LANG=en_US.UTF-8
+LANGUAGE=en_US:en
+LC_ALL=C
+
 for arg in "$@"; do
  process_option $arg
 done

+if [ $no_site_packages -eq 1 ]; then
+  installvenvopts="--no-site-packages"
+fi
+
+function init_testr {
+  if [ ! -d .testrepository ]; then
+    ${wrapper} testr init
+  fi
+}
+
+function run_tests {
+  # Cleanup *pyc
+  ${wrapper} find . -type f -name "*.pyc" -delete
+
+  if [ $coverage -eq 1 ]; then
+    # Do not test test_coverage_ext when gathering coverage.
+    if [ "x$testrargs" = "x" ]; then
+      testrargs="^(?!.*test_coverage_ext).*$"
+    fi
+    export PYTHON="${wrapper} coverage run --source novaclient --parallel-mode"
+  fi
+  # Just run the test suites in current environment
+  set +e
+  TESTRTESTS="$TESTRTESTS $testrargs"
+  echo "Running \`${wrapper} $TESTRTESTS\`"
+  ${wrapper} $TESTRTESTS
+  RESULT=$?
+  set -e
+
+  copy_subunit_log
+
+  return $RESULT
+}
+
+function copy_subunit_log {
+  LOGNAME=`cat .testrepository/next-stream`
+  LOGNAME=$(($LOGNAME - 1))
+  LOGNAME=".testrepository/${LOGNAME}"
+  cp $LOGNAME subunit.log
+}
+
+function run_pep8 {
+  echo "Running pep8 ..."
+  srcfiles="--exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*"
+  srcfiles+=",*egg,build ."
+  # Just run PEP8 in current environment
+  #
+  ignore="--ignore=E12,E711,E721,E712"
+  ${wrapper} pep8 ${ignore} --show-source ${srcfiles}
+}
+
+TESTRTESTS="testr run --parallel $testropts"
+
+if [ $never_venv -eq 0 ]
+then
+  # Remove the virtual environment if --force used
+  if [ $force -eq 1 ]; then
+    echo "Cleaning virtualenv..."
+    rm -rf ${venv}
+  fi
+  if [ -e ${venv} ]; then
+    wrapper="${with_venv}"
+  else
+    if [ $always_venv -eq 1 ]; then
+      # Automatically install the virtualenv
+      python tools/install_venv.py $installvenvopts
+      wrapper="${with_venv}"
+    else
+      echo -e "No virtual environment found...create one? (Y/n) \c"
+      read use_ve
+      if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then
+        # Install the virtualenv and run the test suite in it
+        python tools/install_venv.py $installvenvopts
+        wrapper=${with_venv}
+      fi
+    fi
+  fi
+fi
+
+# Delete old coverage data from previous runs
+if [ $coverage -eq 1 ]; then
+    ${wrapper} coverage erase
+fi
+
 if [ $just_pep8 -eq 1 ]; then
-  tox -e pep8
-  exit
+    run_pep8
+    exit
 fi

-tox -e py27 $toxargs 2>&1 | tee run_tests.err.log  || exit
-if [ ${PIPESTATUS[0]} -ne 0 ]; then
-  exit ${PIPESTATUS[0]}
+init_testr
+run_tests
+
+# NOTE(sirp): we only want to run pep8 when we're running the full-test suite,
+# not when we're running tests individually. To handle this, we need to
+# distinguish between options (noseopts), which begin with a '-', and
+# arguments (testrargs).
+if [ -z "$testrargs" ]; then
+  if [ $no_pep8 -eq 0 ]; then
+    run_pep8
+  fi
 fi

-if [ -z "$toxargs" ]; then
-  tox -e pep8
+if [ $coverage -eq 1 ]; then
+    echo "Generating coverage report in covhtml/"
+    ${wrapper} coverage combine
+    ${wrapper} coverage html --include='novaclient/*' --omit='novaclient/openstack/common/*' -d covhtml -i
 fi
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,11 +1,3 @@
-[nosetests]
-cover-package = ceilometerclient
-cover-html = true
-cover-erase = true
-cover-inclusive = true
-verbosity=2
-detailed-errors=1
-
 [build_sphinx]
 source-dir = doc/source
 build-dir = doc/build
--- a/setup.py
+++ b/setup.py
@@ -24,16 +24,15 @@ def read(fname):
 setuptools.setup(
    name=project,
    version=setup.get_version(project),
-    author='Ceilometer Developers',
+    author='OpenStack',
    author_email='openstack-dev@lists.openstack.org',
    description="Client library for ceilometer",
    long_description=read('README.md'),
-    license='Apache',
+    license="Apache License, Version 2.0",
    url='https://github.com/openstack/python-ceilometerclient',
    packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
    include_package_data=True,
    install_requires=setup.parse_requirements(),
-    test_suite="nose.collector",
    cmdclass=setup.get_cmdclass(),
    classifiers=[
        'Development Status :: 4 - Beta',
--- a/tests/test_shell.py
+++ b/tests/test_shell.py
@@ -1,9 +1,11 @@
 import cStringIO
-import os
 import httplib2
+import re
 import sys

-import mox
+import fixtures
+from testtools import matchers
+
 try:
    import json
 except ImportError:
@@ -15,50 +17,26 @@ from ceilometerclient.v1 import client as v1client
 import ceilometerclient.shell
 from tests import utils

-
-class ShellValidationTest(utils.BaseTestCase):
-
-    def shell_error(self, argstr, error_match):
-        orig = sys.stderr
-        try:
-            sys.stderr = cStringIO.StringIO()
-            _shell = ceilometerclient.shell.CeilometerShell()
-            _shell.main(argstr.split())
-        except exc.CommandError as e:
-            self.assertRegexpMatches(e.__str__(), error_match)
-        else:
-            self.fail('Expected error matching: %s' % error_match)
-        finally:
-            err = sys.stderr.getvalue()
-            sys.stderr.close()
-            sys.stderr = orig
-        return err
+FAKE_ENV = {'OS_USERNAME': 'username',
+            'OS_PASSWORD': 'password',
+            'OS_TENANT_NAME': 'tenant_name',
+            'OS_AUTH_URL': 'http://no.where'}


 class ShellTest(utils.BaseTestCase):
+    re_options = re.DOTALL | re.MULTILINE

    # Patch os.environ to avoid required auth info.
+    def make_env(self, exclude=None):
+        env = dict((k, v) for k, v in FAKE_ENV.items() if k != exclude)
+        self.useFixture(fixtures.MonkeyPatch('os.environ', env))
+
    def setUp(self):
        super(ShellTest, self).setUp()
        self.m.StubOutWithMock(ksclient, 'Client')
        self.m.StubOutWithMock(v1client.Client, 'json_request')
        self.m.StubOutWithMock(v1client.Client, 'raw_request')

-        global _old_env
-        fake_env = {
-            'OS_USERNAME': 'username',
-            'OS_PASSWORD': 'password',
-            'OS_TENANT_NAME': 'tenant_name',
-            'OS_AUTH_URL': 'http://no.where',
-        }
-        _old_env, os.environ = os.environ, fake_env.copy()
-
-    def tearDown(self):
-        super(ShellTest, self).tearDown()
-        self.m.UnsetStubs()
-        global _old_env
-        os.environ = _old_env
-
    def shell(self, argstr):
        orig = sys.stdout
        try:
@@ -85,19 +63,21 @@ class ShellTest(utils.BaseTestCase):

    def test_help(self):
        required = [
-            '^usage: ceilometer',
-            '(?m)^See "ceilometer help COMMAND" '
+            '.*?^usage: ceilometer',
+            '.*?^See "ceilometer help COMMAND" '
            'for help on a specific command',
        ]
        for argstr in ['--help', 'help']:
            help_text = self.shell(argstr)
            for r in required:
-                self.assertRegexpMatches(help_text, r)
+                self.assertThat(help_text,
+                                matchers.MatchesRegex(r,
+                                                      self.re_options))

    def test_help_on_subcommand(self):
        required = [
-            '^usage: ceilometer meter-list',
-            "(?m)^List the user's meter",
+            '.*?^usage: ceilometer meter-list',
+            ".*?^List the user's meter",
        ]
        argstrings = [
            'help meter-list',
@@ -105,19 +85,9 @@ class ShellTest(utils.BaseTestCase):
        for argstr in argstrings:
            help_text = self.shell(argstr)
            for r in required:
-                self.assertRegexpMatches(help_text, r)
+                self.assertThat(help_text,
+                                matchers.MatchesRegex(r, self.re_options))

    def test_auth_param(self):
-        class TokenContext(object):
-            def __enter__(self):
-                fake_env = {
-                    'OS_AUTH_TOKEN': 'token',
-                    'CEILOMETER_URL': 'http://no.where'
-                }
-                self.old_env, os.environ = os.environ, fake_env.copy()
-
-            def __exit__(self, exc_type, exc_value, traceback):
-                os.environ = self.old_env
-
-        with TokenContext():
-            self.test_help()
+        self.make_env(exclude='OS_USERNAME')
+        self.test_help()
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -14,18 +14,21 @@
 #    under the License.

 import copy
+import fixtures
 import StringIO
 import mox
-import unittest2
+import testtools

 from ceilometerclient.common import http


-class BaseTestCase(unittest2.TestCase):
+class BaseTestCase(testtools.TestCase):

    def setUp(self):
        super(BaseTestCase, self).setUp()
        self.m = mox.Mox()
+        self.addCleanup(self.m.UnsetStubs)
+        self.useFixture(fixtures.FakeLogger())


 class FakeAPI(object):
--- a/tools/test-requires
+++ b/tools/test-requires
@@ -1,11 +1,11 @@
 # This file is managed by openstack-depends
+coverage>=3.6
+discover
 distribute>=0.6.24
+fixtures>=0.3.12
 mox>=0.5.3
-nose
-nose-exclude
-nosehtmloutput>=0.0.3
-nosexcover
-openstack.nose_plugin>=0.7
-pep8==1.3.3
+pep8==1.4.5
+python-subunit
 sphinx>=1.1.2
-unittest2
+testrepository>=0.0.13
+testtools>=0.9.29
--- a/tox.ini
+++ b/tox.ini
@@ -2,45 +2,28 @@
 envlist = py26,py27,pep8

 [testenv]
+sitepackages = True
 setenv = VIRTUAL_ENV={envdir}
-         NOSE_WITH_OPENSTACK=1
-         NOSE_OPENSTACK_COLOR=1
-         NOSE_OPENSTACK_RED=0.05
-         NOSE_OPENSTACK_YELLOW=0.025
-         NOSE_OPENSTACK_SHOW_ELAPSED=1
+         LANG=en_US.UTF-8
+         LANGUAGE=en_US:en
+         LC_ALL=C
 deps = -r{toxinidir}/tools/pip-requires
       -r{toxinidir}/tools/test-requires
-commands = nosetests
+commands =
+  python setup.py testr --testr-args='{posargs}'
+
+[tox:jenkins]
+sitepackages = True
+downloadcache = ~/cache/pip

 [testenv:pep8]
-deps = pep8==1.3.3
-commands = pep8 --repeat --show-source ceilometerclient setup.py
+deps = pep8==1.4.5
+commands =
+  pep8 --ignore=E12,E711,E721,E712 --show-source \
+    --exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build .
+
+[testenv:cover]
+commands = python setup.py testr --coverage --testr-args='{posargs}'

 [testenv:venv]
 commands = {posargs}
-
-[testenv:cover]
-commands = nosetests --cover-erase --cover-package=ceilometerclient --with-xcoverage
-
-[tox:jenkins]
-downloadcache = ~/cache/pip
-
-[testenv:jenkins26]
-basepython = python2.6
-setenv = NOSE_WITH_XUNIT=1
-deps = file://{toxinidir}/.cache.bundle
-
-[testenv:jenkins27]
-basepython = python2.7
-setenv = NOSE_WITH_XUNIT=1
-deps = file://{toxinidir}/.cache.bundle
-
-[testenv:jenkinscover]
-deps = file://{toxinidir}/.cache.bundle
-setenv = NOSE_WITH_XUNIT=1
-commands = nosetests --cover-erase --cover-package=ceilometerclient --with-xcoverage
-
-[testenv:jenkinsvenv]
-deps = file://{toxinidir}/.cache.bundle
-setenv = NOSE_WITH_XUNIT=1
-commands = {posargs}