Add unittest framework + tests for wrapper scripts

This commit adds a framework for running unittests on tempest to
verify that tempest works as expect. The first tests added are to
verify the response codes of the test runner wrapper scripts we use
on jenkins jobs.

Change-Id: If8e91238054593999e7b5bc34da499bd0ac02311

doc/source/field_guide/unit_tests.rst

@@ -0,0 +1 @@
+../../../tempest/tests/README.rst

doc/source/index.rst

@@ -32,6 +32,7 @@ where your test contributions should go.
    field_guide/stress
    field_guide/thirdparty
    field_guide/whitebox
+   field_guide/unit_tests
 
 ------------------
 API and test cases

tempest/hacking/checks.py

@@ -53,17 +53,6 @@ def import_no_clients_in_api(physical_line, filename):
                      " in tempest/api/* tests"))
 
 
-def import_no_files_in_tests(physical_line, filename):
-    """Check for merges that try to land into tempest/tests
-
-    T103: tempest/tests directory is deprecated
-    """
-
-    if "tempest/tests" in filename:
-        return (0, ("T103: tempest/tests is deprecated"))
-
-
 def factory(register):
     register(skip_bugs)
     register(import_no_clients_in_api)
-    register(import_no_files_in_tests)

tempest/tests/README.rst

@@ -0,0 +1,25 @@
+Tempest Guide to Unit tests
+===========================
+
+What are these tests?
+---------------------
+
+Unit tests are the self checks for Tempest. They provide functional
+verification and regression checking for the internal components of tempest.
+They should be used to just verify that the individual pieces of tempest are
+working as expected. They should not require an external service to be running
+and should be able to run solely from the tempest tree.
+
+Why are these tests in tempest?
+-------------------------------
+These tests exist to make sure that the mechanisms that we use inside of
+tempest to are valid and remain functional. They are only here for self
+validation of tempest.
+
+
+Scope of these tests
+--------------------
+Unit tests should not require an external service to be running or any extra
+configuration to run. Any state that is required for a test should either be
+mocked out or created in a temporary test directory. (see test_wrappers.py for
+an example of using a temporary test directory)

tempest/tests/files/failing-tests

@@ -0,0 +1,25 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import testtools
+
+class FakeTestClass(testtools.TestCase):
+    def test_pass(self):
+        self.assertTrue(False)
+
+    def test_pass_list(self):
+        test_list = ['test', 'a', 'b']
+        self.assertIn('fail', test_list)

tempest/tests/files/passing-tests

@@ -0,0 +1,25 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import testtools
+
+class FakeTestClass(testtools.TestCase):
+    def test_pass(self):
+        self.assertTrue(True)
+
+    def test_pass_list(self):
+        test_list = ['test', 'a', 'b']
+        self.assertIn('test', test_list)

tempest/tests/files/setup.cfg

@@ -0,0 +1,20 @@
+[metadata]
+name = tempest_unit_tests
+version = 1
+summary = Fake Project for testing wrapper scripts
+author = OpenStack QA
+author-email = openstack-qa@lists.openstack.org
+home-page = http://www.openstack.org/
+classifier =
+    Intended Audience :: Information Technology
+    Intended Audience :: System Administrators
+    Intended Audience :: Developers
+    License :: OSI Approved :: Apache Software License
+    Operating System :: POSIX :: Linux
+    Programming Language :: Python
+    Programming Language :: Python :: 2
+    Programming Language :: Python :: 2.7
+
+[global]
+setup-hooks =
+    pbr.hooks.setup_hook

tempest/tests/files/testr-conf

@@ -0,0 +1,5 @@
+[DEFAULT]
+test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list
+group_regex=([^\.]*\.)*

tempest/tests/test_wrappers.py

@@ -0,0 +1,103 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import shutil
+import subprocess
+import tempfile
+import testtools
+
+from tempest.test import attr
+
+DEVNULL = open(os.devnull, 'wb')
+
+
+class TestWrappers(testtools.TestCase):
+    def setUp(self):
+        super(TestWrappers, self).setUp()
+        # Setup test dirs
+        self.directory = tempfile.mkdtemp(prefix='tempest-unit')
+        self.test_dir = os.path.join(self.directory, 'tests')
+        os.mkdir(self.test_dir)
+        # Setup Test files
+        self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
+        self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
+        self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
+        self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
+        self.init_file = os.path.join(self.test_dir, '__init__.py')
+        self.setup_py = os.path.join(self.directory, 'setup.py')
+        shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
+        shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
+        shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
+        shutil.copy('setup.py', self.setup_py)
+        shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file)
+        shutil.copy('tempest/tests/files/__init__.py', self.init_file)
+
+    @attr(type='smoke')
+    def test_pretty_tox(self):
+        # Copy wrapper script and requirements:
+        pretty_tox = os.path.join(self.directory, 'pretty_tox.sh')
+        shutil.copy('tools/pretty_tox.sh', pretty_tox)
+        # Change directory, run wrapper and check result
+        self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+        os.chdir(self.directory)
+        # Git init is required for the pbr testr command. pbr requires a git
+        # version or an sdist to work. so make the test directory a git repo
+        # too.
+        subprocess.call(['git', 'init'])
+        exit_code = subprocess.call('sh pretty_tox.sh tests.passing',
+                                    shell=True, stdout=DEVNULL, stderr=DEVNULL)
+        self.assertEquals(exit_code, 0)
+
+    @attr(type='smoke')
+    def test_pretty_tox_fails(self):
+        # Copy wrapper script and requirements:
+        pretty_tox = os.path.join(self.directory, 'pretty_tox.sh')
+        shutil.copy('tools/pretty_tox.sh', pretty_tox)
+        # Change directory, run wrapper and check result
+        self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+        os.chdir(self.directory)
+        # Git init is required for the pbr testr command. pbr requires a git
+        # version or an sdist to work. so make the test directory a git repo
+        # too.
+        subprocess.call(['git', 'init'])
+        exit_code = subprocess.call('sh pretty_tox.sh', shell=True,
+                                    stdout=DEVNULL, stderr=DEVNULL)
+        self.assertEquals(exit_code, 1)
+
+    @attr(type='smoke')
+    def test_pretty_tox_serial(self):
+        # Copy wrapper script and requirements:
+        pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh')
+        shutil.copy('tools/pretty_tox_serial.sh', pretty_tox)
+        # Change directory, run wrapper and check result
+        self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+        os.chdir(self.directory)
+        exit_code = subprocess.call('sh pretty_tox_serial.sh tests.passing',
+                                    shell=True, stdout=DEVNULL, stderr=DEVNULL)
+        self.assertEquals(exit_code, 0)
+
+    @attr(type='smoke')
+    def test_pretty_tox_serial_fails(self):
+        # Copy wrapper script and requirements:
+        pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh')
+        shutil.copy('tools/pretty_tox_serial.sh', pretty_tox)
+        # Change directory, run wrapper and check result
+        self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+        os.chdir(self.directory)
+        exit_code = subprocess.call('sh pretty_tox_serial.sh', shell=True,
+                                    stdout=DEVNULL, stderr=DEVNULL)
+        self.assertEquals(exit_code, 1)

tox.ini

@@ -19,13 +19,13 @@ setenv = VIRTUAL_ENV={envdir}
 # The regex below is used to select which tests to run and exclude the slow tag:
 # See the testrepostiory bug: https://bugs.launchpad.net/testrepository/+bug/1208610
 commands =
-  sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}'
+  sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}'
 
 [testenv:testr-full]
 sitepackages = True
 setenv = VIRTUAL_ENV={envdir}
 commands =
-  sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}'
+  sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}'
 
 [testenv:heat-slow]
 sitepackages = True
@@ -44,7 +44,7 @@ setenv = VIRTUAL_ENV={envdir}
          NOSE_OPENSTACK_SHOW_ELAPSED=1
          NOSE_OPENSTACK_STDOUT=1
 commands =
-  nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli {posargs}
+  nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli tempest/tests {posargs}
 
 [testenv:py26-smoke]
 setenv = VIRTUAL_ENV={envdir}
@@ -71,7 +71,7 @@ sitepackages = True
 setenv = VIRTUAL_ENV={envdir}
 commands =
    python -m tools/tempest_coverage -c start --combine
-   sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli))'
+   sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests))'
    python -m tools/tempest_coverage -c report --html {posargs}
 
 [testenv:stress]