From 7091db40a9486ebdc8a19699826b171465e40384 Mon Sep 17 00:00:00 2001 From: Angus Salkeld Date: Fri, 10 May 2013 21:39:34 +1000 Subject: [PATCH] Use testr to run tests This is taken largely from python-novaclient and Heat Change-Id: I8f2eb9185ac073e9ee97bb9c757a4c3c2f8d6d6b --- .gitignore | 2 + .testr.conf | 4 ++ run_tests.sh | 164 ++++++++++++++++++++++++++++++++++++++------ setup.cfg | 8 --- setup.py | 5 +- tests/test_shell.py | 78 +++++++-------------- tests/utils.py | 7 +- tools/test-requires | 14 ++-- tox.ini | 51 +++++--------- 9 files changed, 203 insertions(+), 130 deletions(-) create mode 100644 .testr.conf diff --git a/.gitignore b/.gitignore index 4bb92bf6..2fff79f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ .coverage .venv +.testrepository +subunit.log *,cover cover *.pyc diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..2109af6c --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 ${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/run_tests.sh b/run_tests.sh index fd32f47d..aea24619 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,49 +1,169 @@ #!/bin/bash +set -eu + function usage { echo "Usage: $0 [OPTION]..." - echo "Run python-ceilometerclient's test suite(s)" + echo "Run python-ceilometerclient test suite" echo "" + echo " -V, --virtual-env Always use virtualenv. Install automatically if not present" + echo " -N, --no-virtual-env Don't use virtualenv. Run tests in local environment" + echo " -s, --no-site-packages Isolate the virtualenv from the global Python environment" + echo " -x, --stop Stop running tests after the first error or failure." + echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -p, --pep8 Just run pep8" + echo " -P, --no-pep8 Don't run pep8" + echo " -c, --coverage Generate coverage report" echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" echo "" - echo "This script is deprecated and currently retained for compatibility." - echo 'You can run the full test suite for multiple environments by running "tox".' - echo 'You can run tests for only python 2.7 by running "tox -e py27", or run only' - echo 'the pep8 tests with "tox -e pep8".' + echo "Note: with no options specified, the script will try to run the tests in a virtual environment," + echo " If no virtualenv is found, the script will ask if you would like to create one. If you " + echo " prefer to run tests NOT in a virtual environment, simply pass the -N option." exit } -command -v tox > /dev/null 2>&1 -if [ $? -ne 0 ]; then - echo 'This script requires "tox" to run.' - echo 'You can install it with "pip install tox".' - exit 1; -fi - -just_pep8=0 - function process_option { case "$1" in -h|--help) usage;; - -p|--pep8) let just_pep8=1;; + -V|--virtual-env) always_venv=1; never_venv=0;; + -N|--no-virtual-env) always_venv=0; never_venv=1;; + -s|--no-site-packages) no_site_packages=1;; + -f|--force) force=1;; + -p|--pep8) just_pep8=1;; + -P|--no-pep8) no_pep8=1;; + -c|--coverage) coverage=1;; + -*) testropts="$testropts $1";; + *) testrargs="$testrargs $1" esac } +venv=.venv +with_venv=tools/with_venv.sh +always_venv=0 +never_venv=0 +force=0 +no_site_packages=0 +installvenvopts= +testrargs= +testropts= +wrapper="" +just_pep8=0 +no_pep8=0 +coverage=0 + +LANG=en_US.UTF-8 +LANGUAGE=en_US:en +LC_ALL=C + for arg in "$@"; do process_option $arg done +if [ $no_site_packages -eq 1 ]; then + installvenvopts="--no-site-packages" +fi + +function init_testr { + if [ ! -d .testrepository ]; then + ${wrapper} testr init + fi +} + +function run_tests { + # Cleanup *pyc + ${wrapper} find . -type f -name "*.pyc" -delete + + if [ $coverage -eq 1 ]; then + # Do not test test_coverage_ext when gathering coverage. + if [ "x$testrargs" = "x" ]; then + testrargs="^(?!.*test_coverage_ext).*$" + fi + export PYTHON="${wrapper} coverage run --source novaclient --parallel-mode" + fi + # Just run the test suites in current environment + set +e + TESTRTESTS="$TESTRTESTS $testrargs" + echo "Running \`${wrapper} $TESTRTESTS\`" + ${wrapper} $TESTRTESTS + RESULT=$? + set -e + + copy_subunit_log + + return $RESULT +} + +function copy_subunit_log { + LOGNAME=`cat .testrepository/next-stream` + LOGNAME=$(($LOGNAME - 1)) + LOGNAME=".testrepository/${LOGNAME}" + cp $LOGNAME subunit.log +} + +function run_pep8 { + echo "Running pep8 ..." + srcfiles="--exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*" + srcfiles+=",*egg,build ." + # Just run PEP8 in current environment + # + ignore="--ignore=E12,E711,E721,E712" + ${wrapper} pep8 ${ignore} --show-source ${srcfiles} +} + +TESTRTESTS="testr run --parallel $testropts" + +if [ $never_venv -eq 0 ] +then + # Remove the virtual environment if --force used + if [ $force -eq 1 ]; then + echo "Cleaning virtualenv..." + rm -rf ${venv} + fi + if [ -e ${venv} ]; then + wrapper="${with_venv}" + else + if [ $always_venv -eq 1 ]; then + # Automatically install the virtualenv + python tools/install_venv.py $installvenvopts + wrapper="${with_venv}" + else + echo -e "No virtual environment found...create one? (Y/n) \c" + read use_ve + if [ "x$use_ve" = "xY" -o "x$use_ve" = "x" -o "x$use_ve" = "xy" ]; then + # Install the virtualenv and run the test suite in it + python tools/install_venv.py $installvenvopts + wrapper=${with_venv} + fi + fi + fi +fi + +# Delete old coverage data from previous runs +if [ $coverage -eq 1 ]; then + ${wrapper} coverage erase +fi + if [ $just_pep8 -eq 1 ]; then - tox -e pep8 - exit + run_pep8 + exit fi -tox -e py27 $toxargs 2>&1 | tee run_tests.err.log || exit -if [ ${PIPESTATUS[0]} -ne 0 ]; then - exit ${PIPESTATUS[0]} +init_testr +run_tests + +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (noseopts), which begin with a '-', and +# arguments (testrargs). +if [ -z "$testrargs" ]; then + if [ $no_pep8 -eq 0 ]; then + run_pep8 + fi fi -if [ -z "$toxargs" ]; then - tox -e pep8 +if [ $coverage -eq 1 ]; then + echo "Generating coverage report in covhtml/" + ${wrapper} coverage combine + ${wrapper} coverage html --include='novaclient/*' --omit='novaclient/openstack/common/*' -d covhtml -i fi diff --git a/setup.cfg b/setup.cfg index a0e3d349..11c72013 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,3 @@ -[nosetests] -cover-package = ceilometerclient -cover-html = true -cover-erase = true -cover-inclusive = true -verbosity=2 -detailed-errors=1 - [build_sphinx] source-dir = doc/source build-dir = doc/build diff --git a/setup.py b/setup.py index 4cbacdb3..9ce0ca7b 100644 --- a/setup.py +++ b/setup.py @@ -24,16 +24,15 @@ def read(fname): setuptools.setup( name=project, version=setup.get_version(project), - author='Ceilometer Developers', + author='OpenStack', author_email='openstack-dev@lists.openstack.org', description="Client library for ceilometer", long_description=read('README.md'), - license='Apache', + license="Apache License, Version 2.0", url='https://github.com/openstack/python-ceilometerclient', packages=setuptools.find_packages(exclude=['tests', 'tests.*']), include_package_data=True, install_requires=setup.parse_requirements(), - test_suite="nose.collector", cmdclass=setup.get_cmdclass(), classifiers=[ 'Development Status :: 4 - Beta', diff --git a/tests/test_shell.py b/tests/test_shell.py index e0b1e57e..14484037 100644 --- a/tests/test_shell.py +++ b/tests/test_shell.py @@ -1,9 +1,11 @@ import cStringIO -import os import httplib2 +import re import sys -import mox +import fixtures +from testtools import matchers + try: import json except ImportError: @@ -15,50 +17,26 @@ from ceilometerclient.v1 import client as v1client import ceilometerclient.shell from tests import utils - -class ShellValidationTest(utils.BaseTestCase): - - def shell_error(self, argstr, error_match): - orig = sys.stderr - try: - sys.stderr = cStringIO.StringIO() - _shell = ceilometerclient.shell.CeilometerShell() - _shell.main(argstr.split()) - except exc.CommandError as e: - self.assertRegexpMatches(e.__str__(), error_match) - else: - self.fail('Expected error matching: %s' % error_match) - finally: - err = sys.stderr.getvalue() - sys.stderr.close() - sys.stderr = orig - return err +FAKE_ENV = {'OS_USERNAME': 'username', + 'OS_PASSWORD': 'password', + 'OS_TENANT_NAME': 'tenant_name', + 'OS_AUTH_URL': 'http://no.where'} class ShellTest(utils.BaseTestCase): + re_options = re.DOTALL | re.MULTILINE # Patch os.environ to avoid required auth info. + def make_env(self, exclude=None): + env = dict((k, v) for k, v in FAKE_ENV.items() if k != exclude) + self.useFixture(fixtures.MonkeyPatch('os.environ', env)) + def setUp(self): super(ShellTest, self).setUp() self.m.StubOutWithMock(ksclient, 'Client') self.m.StubOutWithMock(v1client.Client, 'json_request') self.m.StubOutWithMock(v1client.Client, 'raw_request') - global _old_env - fake_env = { - 'OS_USERNAME': 'username', - 'OS_PASSWORD': 'password', - 'OS_TENANT_NAME': 'tenant_name', - 'OS_AUTH_URL': 'http://no.where', - } - _old_env, os.environ = os.environ, fake_env.copy() - - def tearDown(self): - super(ShellTest, self).tearDown() - self.m.UnsetStubs() - global _old_env - os.environ = _old_env - def shell(self, argstr): orig = sys.stdout try: @@ -85,19 +63,21 @@ class ShellTest(utils.BaseTestCase): def test_help(self): required = [ - '^usage: ceilometer', - '(?m)^See "ceilometer help COMMAND" ' + '.*?^usage: ceilometer', + '.*?^See "ceilometer help COMMAND" ' 'for help on a specific command', ] for argstr in ['--help', 'help']: help_text = self.shell(argstr) for r in required: - self.assertRegexpMatches(help_text, r) + self.assertThat(help_text, + matchers.MatchesRegex(r, + self.re_options)) def test_help_on_subcommand(self): required = [ - '^usage: ceilometer meter-list', - "(?m)^List the user's meter", + '.*?^usage: ceilometer meter-list', + ".*?^List the user's meter", ] argstrings = [ 'help meter-list', @@ -105,19 +85,9 @@ class ShellTest(utils.BaseTestCase): for argstr in argstrings: help_text = self.shell(argstr) for r in required: - self.assertRegexpMatches(help_text, r) + self.assertThat(help_text, + matchers.MatchesRegex(r, self.re_options)) def test_auth_param(self): - class TokenContext(object): - def __enter__(self): - fake_env = { - 'OS_AUTH_TOKEN': 'token', - 'CEILOMETER_URL': 'http://no.where' - } - self.old_env, os.environ = os.environ, fake_env.copy() - - def __exit__(self, exc_type, exc_value, traceback): - os.environ = self.old_env - - with TokenContext(): - self.test_help() + self.make_env(exclude='OS_USERNAME') + self.test_help() diff --git a/tests/utils.py b/tests/utils.py index 77b0f9e9..f0026e2c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -14,18 +14,21 @@ # under the License. import copy +import fixtures import StringIO import mox -import unittest2 +import testtools from ceilometerclient.common import http -class BaseTestCase(unittest2.TestCase): +class BaseTestCase(testtools.TestCase): def setUp(self): super(BaseTestCase, self).setUp() self.m = mox.Mox() + self.addCleanup(self.m.UnsetStubs) + self.useFixture(fixtures.FakeLogger()) class FakeAPI(object): diff --git a/tools/test-requires b/tools/test-requires index 54b19cc3..06f5bd97 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,11 +1,11 @@ # This file is managed by openstack-depends +coverage>=3.6 +discover distribute>=0.6.24 +fixtures>=0.3.12 mox>=0.5.3 -nose -nose-exclude -nosehtmloutput>=0.0.3 -nosexcover -openstack.nose_plugin>=0.7 -pep8==1.3.3 +pep8==1.4.5 +python-subunit sphinx>=1.1.2 -unittest2 +testrepository>=0.0.13 +testtools>=0.9.29 diff --git a/tox.ini b/tox.ini index 3ea8a2f6..27c8600a 100644 --- a/tox.ini +++ b/tox.ini @@ -2,45 +2,28 @@ envlist = py26,py27,pep8 [testenv] +sitepackages = True setenv = VIRTUAL_ENV={envdir} - NOSE_WITH_OPENSTACK=1 - NOSE_OPENSTACK_COLOR=1 - NOSE_OPENSTACK_RED=0.05 - NOSE_OPENSTACK_YELLOW=0.025 - NOSE_OPENSTACK_SHOW_ELAPSED=1 + LANG=en_US.UTF-8 + LANGUAGE=en_US:en + LC_ALL=C deps = -r{toxinidir}/tools/pip-requires -r{toxinidir}/tools/test-requires -commands = nosetests +commands = + python setup.py testr --testr-args='{posargs}' + +[tox:jenkins] +sitepackages = True +downloadcache = ~/cache/pip [testenv:pep8] -deps = pep8==1.3.3 -commands = pep8 --repeat --show-source ceilometerclient setup.py +deps = pep8==1.4.5 +commands = + pep8 --ignore=E12,E711,E721,E712 --show-source \ + --exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build . + +[testenv:cover] +commands = python setup.py testr --coverage --testr-args='{posargs}' [testenv:venv] commands = {posargs} - -[testenv:cover] -commands = nosetests --cover-erase --cover-package=ceilometerclient --with-xcoverage - -[tox:jenkins] -downloadcache = ~/cache/pip - -[testenv:jenkins26] -basepython = python2.6 -setenv = NOSE_WITH_XUNIT=1 -deps = file://{toxinidir}/.cache.bundle - -[testenv:jenkins27] -basepython = python2.7 -setenv = NOSE_WITH_XUNIT=1 -deps = file://{toxinidir}/.cache.bundle - -[testenv:jenkinscover] -deps = file://{toxinidir}/.cache.bundle -setenv = NOSE_WITH_XUNIT=1 -commands = nosetests --cover-erase --cover-package=ceilometerclient --with-xcoverage - -[testenv:jenkinsvenv] -deps = file://{toxinidir}/.cache.bundle -setenv = NOSE_WITH_XUNIT=1 -commands = {posargs}