diff --git a/doc/source/field_guide/unit_tests.rst b/doc/source/field_guide/unit_tests.rst new file mode 120000 index 0000000000..67a8b2035d --- /dev/null +++ b/doc/source/field_guide/unit_tests.rst @@ -0,0 +1 @@ +../../../tempest/tests/README.rst \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 00c4e9a2c0..f70cdd10de 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,6 +32,7 @@ where your test contributions should go. field_guide/stress field_guide/thirdparty field_guide/whitebox + field_guide/unit_tests ------------------ API and test cases diff --git a/tempest/hacking/checks.py b/tempest/hacking/checks.py index f9eb968be1..8cfd548234 100644 --- a/tempest/hacking/checks.py +++ b/tempest/hacking/checks.py @@ -53,17 +53,6 @@ def import_no_clients_in_api(physical_line, filename): " in tempest/api/* tests")) -def import_no_files_in_tests(physical_line, filename): - """Check for merges that try to land into tempest/tests - - T103: tempest/tests directory is deprecated - """ - - if "tempest/tests" in filename: - return (0, ("T103: tempest/tests is deprecated")) - - def factory(register): register(skip_bugs) register(import_no_clients_in_api) - register(import_no_files_in_tests) diff --git a/tempest/tests/README.rst b/tempest/tests/README.rst new file mode 100644 index 0000000000..4098686527 --- /dev/null +++ b/tempest/tests/README.rst @@ -0,0 +1,25 @@ +Tempest Guide to Unit tests +=========================== + +What are these tests? +--------------------- + +Unit tests are the self checks for Tempest. They provide functional +verification and regression checking for the internal components of tempest. +They should be used to just verify that the individual pieces of tempest are +working as expected. They should not require an external service to be running +and should be able to run solely from the tempest tree. + +Why are these tests in tempest? +------------------------------- +These tests exist to make sure that the mechanisms that we use inside of +tempest to are valid and remain functional. They are only here for self +validation of tempest. + + +Scope of these tests +-------------------- +Unit tests should not require an external service to be running or any extra +configuration to run. Any state that is required for a test should either be +mocked out or created in a temporary test directory. (see test_wrappers.py for +an example of using a temporary test directory) diff --git a/tempest/tests/__init__.py b/tempest/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tempest/tests/files/__init__.py b/tempest/tests/files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tempest/tests/files/failing-tests b/tempest/tests/files/failing-tests new file mode 100644 index 0000000000..0ec5421f56 --- /dev/null +++ b/tempest/tests/files/failing-tests @@ -0,0 +1,25 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +class FakeTestClass(testtools.TestCase): + def test_pass(self): + self.assertTrue(False) + + def test_pass_list(self): + test_list = ['test', 'a', 'b'] + self.assertIn('fail', test_list) diff --git a/tempest/tests/files/passing-tests b/tempest/tests/files/passing-tests new file mode 100644 index 0000000000..2f5b7c96c0 --- /dev/null +++ b/tempest/tests/files/passing-tests @@ -0,0 +1,25 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import testtools + +class FakeTestClass(testtools.TestCase): + def test_pass(self): + self.assertTrue(True) + + def test_pass_list(self): + test_list = ['test', 'a', 'b'] + self.assertIn('test', test_list) diff --git a/tempest/tests/files/setup.cfg b/tempest/tests/files/setup.cfg new file mode 100644 index 0000000000..8639baaf7a --- /dev/null +++ b/tempest/tests/files/setup.cfg @@ -0,0 +1,20 @@ +[metadata] +name = tempest_unit_tests +version = 1 +summary = Fake Project for testing wrapper scripts +author = OpenStack QA +author-email = openstack-qa@lists.openstack.org +home-page = http://www.openstack.org/ +classifier = + Intended Audience :: Information Technology + Intended Audience :: System Administrators + Intended Audience :: Developers + License :: OSI Approved :: Apache Software License + Operating System :: POSIX :: Linux + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + +[global] +setup-hooks = + pbr.hooks.setup_hook diff --git a/tempest/tests/files/testr-conf b/tempest/tests/files/testr-conf new file mode 100644 index 0000000000..d5ad083225 --- /dev/null +++ b/tempest/tests/files/testr-conf @@ -0,0 +1,5 @@ +[DEFAULT] +test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list +group_regex=([^\.]*\.)* diff --git a/tempest/tests/test_wrappers.py b/tempest/tests/test_wrappers.py new file mode 100644 index 0000000000..aeea98dd40 --- /dev/null +++ b/tempest/tests/test_wrappers.py @@ -0,0 +1,103 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import shutil +import subprocess +import tempfile +import testtools + +from tempest.test import attr + +DEVNULL = open(os.devnull, 'wb') + + +class TestWrappers(testtools.TestCase): + def setUp(self): + super(TestWrappers, self).setUp() + # Setup test dirs + self.directory = tempfile.mkdtemp(prefix='tempest-unit') + self.test_dir = os.path.join(self.directory, 'tests') + os.mkdir(self.test_dir) + # Setup Test files + self.testr_conf_file = os.path.join(self.directory, '.testr.conf') + self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg') + self.passing_file = os.path.join(self.test_dir, 'test_passing.py') + self.failing_file = os.path.join(self.test_dir, 'test_failing.py') + self.init_file = os.path.join(self.test_dir, '__init__.py') + self.setup_py = os.path.join(self.directory, 'setup.py') + shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file) + shutil.copy('tempest/tests/files/passing-tests', self.passing_file) + shutil.copy('tempest/tests/files/failing-tests', self.failing_file) + shutil.copy('setup.py', self.setup_py) + shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file) + shutil.copy('tempest/tests/files/__init__.py', self.init_file) + + @attr(type='smoke') + def test_pretty_tox(self): + # Copy wrapper script and requirements: + pretty_tox = os.path.join(self.directory, 'pretty_tox.sh') + shutil.copy('tools/pretty_tox.sh', pretty_tox) + # Change directory, run wrapper and check result + self.addCleanup(os.chdir, os.path.abspath(os.curdir)) + os.chdir(self.directory) + # Git init is required for the pbr testr command. pbr requires a git + # version or an sdist to work. so make the test directory a git repo + # too. + subprocess.call(['git', 'init']) + exit_code = subprocess.call('sh pretty_tox.sh tests.passing', + shell=True, stdout=DEVNULL, stderr=DEVNULL) + self.assertEquals(exit_code, 0) + + @attr(type='smoke') + def test_pretty_tox_fails(self): + # Copy wrapper script and requirements: + pretty_tox = os.path.join(self.directory, 'pretty_tox.sh') + shutil.copy('tools/pretty_tox.sh', pretty_tox) + # Change directory, run wrapper and check result + self.addCleanup(os.chdir, os.path.abspath(os.curdir)) + os.chdir(self.directory) + # Git init is required for the pbr testr command. pbr requires a git + # version or an sdist to work. so make the test directory a git repo + # too. + subprocess.call(['git', 'init']) + exit_code = subprocess.call('sh pretty_tox.sh', shell=True, + stdout=DEVNULL, stderr=DEVNULL) + self.assertEquals(exit_code, 1) + + @attr(type='smoke') + def test_pretty_tox_serial(self): + # Copy wrapper script and requirements: + pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh') + shutil.copy('tools/pretty_tox_serial.sh', pretty_tox) + # Change directory, run wrapper and check result + self.addCleanup(os.chdir, os.path.abspath(os.curdir)) + os.chdir(self.directory) + exit_code = subprocess.call('sh pretty_tox_serial.sh tests.passing', + shell=True, stdout=DEVNULL, stderr=DEVNULL) + self.assertEquals(exit_code, 0) + + @attr(type='smoke') + def test_pretty_tox_serial_fails(self): + # Copy wrapper script and requirements: + pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh') + shutil.copy('tools/pretty_tox_serial.sh', pretty_tox) + # Change directory, run wrapper and check result + self.addCleanup(os.chdir, os.path.abspath(os.curdir)) + os.chdir(self.directory) + exit_code = subprocess.call('sh pretty_tox_serial.sh', shell=True, + stdout=DEVNULL, stderr=DEVNULL) + self.assertEquals(exit_code, 1) diff --git a/tox.ini b/tox.ini index dc4873569e..a101a9e305 100644 --- a/tox.ini +++ b/tox.ini @@ -19,13 +19,13 @@ setenv = VIRTUAL_ENV={envdir} # The regex below is used to select which tests to run and exclude the slow tag: # See the testrepostiory bug: https://bugs.launchpad.net/testrepository/+bug/1208610 commands = - sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}' + sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}' [testenv:testr-full] sitepackages = True setenv = VIRTUAL_ENV={envdir} commands = - sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}' + sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}' [testenv:heat-slow] sitepackages = True @@ -44,7 +44,7 @@ setenv = VIRTUAL_ENV={envdir} NOSE_OPENSTACK_SHOW_ELAPSED=1 NOSE_OPENSTACK_STDOUT=1 commands = - nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli {posargs} + nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli tempest/tests {posargs} [testenv:py26-smoke] setenv = VIRTUAL_ENV={envdir} @@ -71,7 +71,7 @@ sitepackages = True setenv = VIRTUAL_ENV={envdir} commands = python -m tools/tempest_coverage -c start --combine - sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli))' + sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests))' python -m tools/tempest_coverage -c report --html {posargs} [testenv:stress]