Add unittest framework + tests for wrapper scripts

This commit adds a framework for running unittests on tempest to
verify that tempest works as expect. The first tests added are to
verify the response codes of the test runner wrapper scripts we use
on jenkins jobs.

Change-Id: If8e91238054593999e7b5bc34da499bd0ac02311
This commit is contained in:
Matthew Treinish 2013-08-13 11:59:06 -04:00
parent 59f5cde931
commit 32d3570c18
12 changed files with 209 additions and 15 deletions

View File

@ -0,0 +1 @@
../../../tempest/tests/README.rst

View File

@ -32,6 +32,7 @@ where your test contributions should go.
field_guide/stress field_guide/stress
field_guide/thirdparty field_guide/thirdparty
field_guide/whitebox field_guide/whitebox
field_guide/unit_tests
------------------ ------------------
API and test cases API and test cases

View File

@ -53,17 +53,6 @@ def import_no_clients_in_api(physical_line, filename):
" in tempest/api/* tests")) " in tempest/api/* tests"))
def import_no_files_in_tests(physical_line, filename):
"""Check for merges that try to land into tempest/tests
T103: tempest/tests directory is deprecated
"""
if "tempest/tests" in filename:
return (0, ("T103: tempest/tests is deprecated"))
def factory(register): def factory(register):
register(skip_bugs) register(skip_bugs)
register(import_no_clients_in_api) register(import_no_clients_in_api)
register(import_no_files_in_tests)

25
tempest/tests/README.rst Normal file
View File

@ -0,0 +1,25 @@
Tempest Guide to Unit tests
===========================
What are these tests?
---------------------
Unit tests are the self checks for Tempest. They provide functional
verification and regression checking for the internal components of tempest.
They should be used to just verify that the individual pieces of tempest are
working as expected. They should not require an external service to be running
and should be able to run solely from the tempest tree.
Why are these tests in tempest?
-------------------------------
These tests exist to make sure that the mechanisms that we use inside of
tempest to are valid and remain functional. They are only here for self
validation of tempest.
Scope of these tests
--------------------
Unit tests should not require an external service to be running or any extra
configuration to run. Any state that is required for a test should either be
mocked out or created in a temporary test directory. (see test_wrappers.py for
an example of using a temporary test directory)

View File

View File

View File

@ -0,0 +1,25 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
class FakeTestClass(testtools.TestCase):
def test_pass(self):
self.assertTrue(False)
def test_pass_list(self):
test_list = ['test', 'a', 'b']
self.assertIn('fail', test_list)

View File

@ -0,0 +1,25 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import testtools
class FakeTestClass(testtools.TestCase):
def test_pass(self):
self.assertTrue(True)
def test_pass_list(self):
test_list = ['test', 'a', 'b']
self.assertIn('test', test_list)

View File

@ -0,0 +1,20 @@
[metadata]
name = tempest_unit_tests
version = 1
summary = Fake Project for testing wrapper scripts
author = OpenStack QA
author-email = openstack-qa@lists.openstack.org
home-page = http://www.openstack.org/
classifier =
Intended Audience :: Information Technology
Intended Audience :: System Administrators
Intended Audience :: Developers
License :: OSI Approved :: Apache Software License
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
[global]
setup-hooks =
pbr.hooks.setup_hook

View File

@ -0,0 +1,5 @@
[DEFAULT]
test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list
group_regex=([^\.]*\.)*

View File

@ -0,0 +1,103 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import shutil
import subprocess
import tempfile
import testtools
from tempest.test import attr
DEVNULL = open(os.devnull, 'wb')
class TestWrappers(testtools.TestCase):
def setUp(self):
super(TestWrappers, self).setUp()
# Setup test dirs
self.directory = tempfile.mkdtemp(prefix='tempest-unit')
self.test_dir = os.path.join(self.directory, 'tests')
os.mkdir(self.test_dir)
# Setup Test files
self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
self.init_file = os.path.join(self.test_dir, '__init__.py')
self.setup_py = os.path.join(self.directory, 'setup.py')
shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
shutil.copy('setup.py', self.setup_py)
shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file)
shutil.copy('tempest/tests/files/__init__.py', self.init_file)
@attr(type='smoke')
def test_pretty_tox(self):
# Copy wrapper script and requirements:
pretty_tox = os.path.join(self.directory, 'pretty_tox.sh')
shutil.copy('tools/pretty_tox.sh', pretty_tox)
# Change directory, run wrapper and check result
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)
# Git init is required for the pbr testr command. pbr requires a git
# version or an sdist to work. so make the test directory a git repo
# too.
subprocess.call(['git', 'init'])
exit_code = subprocess.call('sh pretty_tox.sh tests.passing',
shell=True, stdout=DEVNULL, stderr=DEVNULL)
self.assertEquals(exit_code, 0)
@attr(type='smoke')
def test_pretty_tox_fails(self):
# Copy wrapper script and requirements:
pretty_tox = os.path.join(self.directory, 'pretty_tox.sh')
shutil.copy('tools/pretty_tox.sh', pretty_tox)
# Change directory, run wrapper and check result
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)
# Git init is required for the pbr testr command. pbr requires a git
# version or an sdist to work. so make the test directory a git repo
# too.
subprocess.call(['git', 'init'])
exit_code = subprocess.call('sh pretty_tox.sh', shell=True,
stdout=DEVNULL, stderr=DEVNULL)
self.assertEquals(exit_code, 1)
@attr(type='smoke')
def test_pretty_tox_serial(self):
# Copy wrapper script and requirements:
pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh')
shutil.copy('tools/pretty_tox_serial.sh', pretty_tox)
# Change directory, run wrapper and check result
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)
exit_code = subprocess.call('sh pretty_tox_serial.sh tests.passing',
shell=True, stdout=DEVNULL, stderr=DEVNULL)
self.assertEquals(exit_code, 0)
@attr(type='smoke')
def test_pretty_tox_serial_fails(self):
# Copy wrapper script and requirements:
pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh')
shutil.copy('tools/pretty_tox_serial.sh', pretty_tox)
# Change directory, run wrapper and check result
self.addCleanup(os.chdir, os.path.abspath(os.curdir))
os.chdir(self.directory)
exit_code = subprocess.call('sh pretty_tox_serial.sh', shell=True,
stdout=DEVNULL, stderr=DEVNULL)
self.assertEquals(exit_code, 1)

View File

@ -19,13 +19,13 @@ setenv = VIRTUAL_ENV={envdir}
# The regex below is used to select which tests to run and exclude the slow tag: # The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepostiory bug: https://bugs.launchpad.net/testrepository/+bug/1208610 # See the testrepostiory bug: https://bugs.launchpad.net/testrepository/+bug/1208610
commands = commands =
sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}' sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}'
[testenv:testr-full] [testenv:testr-full]
sitepackages = True sitepackages = True
setenv = VIRTUAL_ENV={envdir} setenv = VIRTUAL_ENV={envdir}
commands = commands =
sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}' sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}'
[testenv:heat-slow] [testenv:heat-slow]
sitepackages = True sitepackages = True
@ -44,7 +44,7 @@ setenv = VIRTUAL_ENV={envdir}
NOSE_OPENSTACK_SHOW_ELAPSED=1 NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1 NOSE_OPENSTACK_STDOUT=1
commands = commands =
nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli {posargs} nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli tempest/tests {posargs}
[testenv:py26-smoke] [testenv:py26-smoke]
setenv = VIRTUAL_ENV={envdir} setenv = VIRTUAL_ENV={envdir}
@ -71,7 +71,7 @@ sitepackages = True
setenv = VIRTUAL_ENV={envdir} setenv = VIRTUAL_ENV={envdir}
commands = commands =
python -m tools/tempest_coverage -c start --combine python -m tools/tempest_coverage -c start --combine
sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli))' sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests))'
python -m tools/tempest_coverage -c report --html {posargs} python -m tools/tempest_coverage -c report --html {posargs}
[testenv:stress] [testenv:stress]