setUpClass/tearDownClass full chain

Ensure the setUpClass/tearDownClass methods calls the super properly.
The setUpClass call tested at setUp time, the tearDownClass at exit
time.

super calling issues are also fixed.

Implements blueprint stop-leaking
Closes-Bug: #1212071

Change-Id: I266efe0cd363b330b1511e7e983dab9f62778848

tempest/api/compute/base.py

@@ -36,6 +36,7 @@ class BaseComputeTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BaseComputeTest, cls).setUpClass()
         if not cls.config.service_available.nova:
             skip_msg = ("%s skipped as nova is not available" % cls.__name__)
             raise cls.skipException(skip_msg)

tempest/api/identity/admin/v3/test_endpoints.py

@@ -53,6 +53,7 @@ class EndPointsTestJSON(base.BaseIdentityAdminTest):
             cls.client.delete_endpoint(e['id'])
         for s in cls.service_ids:
             cls.identity_client.delete_service(s)
+        super(EndPointsTestJSON, cls).tearDownClass()
 
     @attr(type='gate')
     def test_list_endpoints(self):

tempest/api/identity/base.py

@@ -25,6 +25,7 @@ class BaseIdentityAdminTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BaseIdentityAdminTest, cls).setUpClass()
         os = clients.AdminManager(interface=cls._interface)
         cls.client = os.identity_client
         cls.token_client = os.token_client
@@ -45,6 +46,7 @@ class BaseIdentityAdminTest(tempest.test.BaseTestCase):
     @classmethod
     def tearDownClass(cls):
         cls.data.teardown_all()
+        super(BaseIdentityAdminTest, cls).tearDownClass()
 
     def disable_user(self, user_name):
         user = self.get_user_by_name(user_name)

tempest/api/image/base.py

@@ -29,6 +29,7 @@ class BaseImageTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BaseImageTest, cls).setUpClass()
         cls.created_images = []
         cls._interface = 'json'
         cls.isolated_creds = isolated_creds.IsolatedCreds(cls.__name__)

tempest/api/network/base.py

@@ -47,6 +47,7 @@ class BaseNetworkTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BaseNetworkTest, cls).setUpClass()
         os = clients.Manager()
         cls.network_cfg = os.config.network
         if not cls.config.service_available.neutron:
@@ -64,6 +65,7 @@ class BaseNetworkTest(tempest.test.BaseTestCase):
             cls.client.delete_subnet(subnet['id'])
         for network in cls.networks:
             cls.client.delete_network(network['id'])
+        super(BaseNetworkTest, cls).tearDownClass()
 
     @classmethod
     def create_network(cls, network_name=None):

tempest/api/object_storage/base.py

@@ -26,6 +26,7 @@ class BaseObjectTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BaseObjectTest, cls).setUpClass()
         if not cls.config.service_available.swift:
             skip_msg = ("%s skipped as swift is not available" % cls.__name__)
             raise cls.skipException(skip_msg)

tempest/api/object_storage/test_container_services.py

@@ -31,6 +31,7 @@ class ContainerTest(base.BaseObjectTest):
     @classmethod
     def tearDownClass(cls):
         cls.delete_containers(cls.containers)
+        super(ContainerTest, cls).tearDownClass()
 
     @attr(type='smoke')
     def test_create_container(self):

tempest/api/object_storage/test_container_sync.py

@@ -50,6 +50,7 @@ class ContainerSyncTest(base.BaseObjectTest):
     def tearDownClass(cls):
         for client in cls.clients.values():
             cls.delete_containers(cls.containers, client[0], client[1])
+        super(ContainerSyncTest, cls).tearDownClass()
 
     @testtools.skip('Until Bug #1093743 is resolved.')
     @attr(type='gate')

tempest/api/object_storage/test_object_expiry.py

@@ -41,6 +41,7 @@ class ObjectExpiryTest(base.BaseObjectTest):
         NotFound exception and also non empty container cannot be deleted.
         """
         cls.delete_containers([cls.container_name])
+        super(ObjectExpiryTest, cls).tearDownClass()
 
     @testtools.skip('Until Bug #1069849 is resolved.')
     @attr(type='gate')

tempest/api/object_storage/test_object_services.py

@@ -47,6 +47,7 @@ class ObjectTest(base.BaseObjectTest):
         cls.delete_containers(cls.containers)
         # delete the user setup created
         cls.data.teardown_all()
+        super(ObjectTest, cls).tearDownClass()
 
     @attr(type='smoke')
     def test_create_object(self):

tempest/api/object_storage/test_object_version.py

@@ -29,6 +29,7 @@ class ContainerTest(base.BaseObjectTest):
     @classmethod
     def tearDownClass(cls):
         cls.delete_containers(cls.containers)
+        super(ContainerTest, cls).tearDownClass()
 
     def assertContainer(self, container, count, byte, versioned):
         resp, _ = self.container_client.list_container_metadata(container)

tempest/api/orchestration/base.py

@@ -28,7 +28,7 @@ class BaseOrchestrationTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
-
+        super(BaseOrchestrationTest, cls).setUpClass()
         os = clients.OrchestrationManager()
         cls.orchestration_cfg = os.config.orchestration
         if not os.config.service_available.heat:
@@ -107,6 +107,7 @@ class BaseOrchestrationTest(tempest.test.BaseTestCase):
     def tearDownClass(cls):
         cls.clear_stacks()
         cls.clear_keypairs()
+        super(BaseOrchestrationTest, cls).tearDownClass()
 
     def wait_for(self, condition):
         """Repeatedly calls condition() until a timeout."""

tempest/api/volume/base.py

@@ -31,6 +31,7 @@ class BaseVolumeTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BaseVolumeTest, cls).setUpClass()
         cls.isolated_creds = isolated_creds.IsolatedCreds(cls.__name__)
 
         if not cls.config.service_available.cinder:

tempest/scenario/manager.py

@@ -150,6 +150,7 @@ class OfficialClientTest(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(OfficialClientTest, cls).setUpClass()
         cls.isolated_creds = isolated_creds.IsolatedCreds(
             __name__, tempest_client=False)
         if cls.config.compute.allow_tenant_isolation:

tempest/test.py

@@ -15,6 +15,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import atexit
 import os
 import time
 
@@ -91,6 +92,17 @@ if 'TEMPEST_PY26_NOSE_COMPAT' in os.environ:
         LOG.info("Overriding skipException to nose SkipTest")
         testtools.TestCase.skipException = nose.plugins.skip.SkipTest
 
+at_exit_set = set()
+
+
+def validate_tearDownClass():
+    if at_exit_set:
+        raise RuntimeError("tearDownClass does not calls the super's"
+                           "tearDownClass in these classes: "
+                           + str(at_exit_set))
+
+atexit.register(validate_tearDownClass)
+
 
 class BaseTestCase(testtools.TestCase,
                    testtools.testcase.WithAttributes,
@@ -98,29 +110,43 @@ class BaseTestCase(testtools.TestCase,
 
     config = config.TempestConfig()
 
+    setUpClassCalled = False
+
     @classmethod
     def setUpClass(cls):
         if hasattr(super(BaseTestCase, cls), 'setUpClass'):
             super(BaseTestCase, cls).setUpClass()
+        cls.setUpClassCalled = True
 
-    def setUp(cls):
-        super(BaseTestCase, cls).setUp()
+    @classmethod
+    def tearDownClass(cls):
+        at_exit_set.remove(cls)
+        if hasattr(super(BaseTestCase, cls), 'tearDownClass'):
+            super(BaseTestCase, cls).tearDownClass()
+
+    def setUp(self):
+        super(BaseTestCase, self).setUp()
+        if not self.setUpClassCalled:
+            raise RuntimeError("setUpClass does not calls the super's"
+                               "setUpClass in the "
+                               + self.__class__.__name__)
+        at_exit_set.add(self.__class__)
         test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
         try:
             test_timeout = int(test_timeout)
         except ValueError:
             test_timeout = 0
         if test_timeout > 0:
-            cls.useFixture(fixtures.Timeout(test_timeout, gentle=True))
+            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
 
         if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
                 os.environ.get('OS_STDOUT_CAPTURE') == '1'):
-            stdout = cls.useFixture(fixtures.StringStream('stdout')).stream
-            cls.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
+            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
         if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
                 os.environ.get('OS_STDERR_CAPTURE') == '1'):
-            stderr = cls.useFixture(fixtures.StringStream('stderr')).stream
-            cls.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
+            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
+            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
 
     @classmethod
     def _get_identity_admin_client(cls):

tempest/thirdparty/boto/test.py

@@ -197,6 +197,7 @@ class BotoTestCase(tempest.test.BaseTestCase):
 
     @classmethod
     def setUpClass(cls):
+        super(BotoTestCase, cls).setUpClass()
         # The trash contains cleanup functions and paramaters in tuples
         # (function, *args, **kwargs)
         cls._resource_trash_bin = {}
@@ -261,6 +262,10 @@ class BotoTestCase(tempest.test.BaseTestCase):
                 LOG.exception(exc)
             finally:
                 del cls._resource_trash_bin[key]
+        super(BotoTestCase, cls).tearDownClass()
+        # NOTE(afazekas): let the super called even on exceptions
+        # The real exceptions already logged, if the super throws another,
+        # does not causes hidden issues
         if fail_count:
             raise exceptions.TearDownException(num=fail_count)