Add workaround to handle the testtool skip exception in CLI test

This commit apply the workaround for testtool skip exception
in CLI test base class.
- https://review.opendev.org/#/c/681340/6

I have modified the workaround done in above commit to more generic
one.

We are good on skipException from test code. That is handled by
teststools.TestCase's self.exception_handlers
- f51ce5f934/testtools/testcase.py (L275)
Mapped handler _report_skip() will add the respective tests to skip list

Current CLI test failure
-https://zuul.opendev.org/t/openstack/build/c1bb61121e1d4d108c52adad200e8991/log/job-output.txt#6354

I have tested both test.BaseTestCase and lib.base.BaseTestCase with all 8 combinations
of below:
 - py >= 3.5 and py < 3.5 (on py2.7)
 - stestr >= 2.50 and stestr < 2.5.0
 - skip exception from setUpClass and skip exception from test code

Closes-Bug: 1847749

Change-Id: Ib70bdffaf4d38743e7cbbeb88af51bb57ceeedf6

releasenotes/notes/fix-1847749-2670b1d4f6097a1a.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Bug#1847749. This privides the workaround of Skip Exception raised instead of skipping
+    the CLI tests. If you are running Tempest with stestr > 2.5.0 then use this fix.
+    Ref- https://github.com/testing-cabal/testtools/issues/272

tempest/lib/base.py

@@ -14,11 +14,29 @@
 #    under the License.
 
 import os
+import sys
 
 import fixtures
+import pkg_resources
 import testtools
 
 
+def _handle_skip_exception():
+    try:
+        stestr_version = pkg_resources.parse_version(
+            pkg_resources.get_distribution("stestr").version)
+        stestr_min = pkg_resources.parse_version('2.5.0')
+        new_stestr = (stestr_version >= stestr_min)
+        import unittest
+        import unittest2
+        if sys.version_info >= (3, 5) and new_stestr:
+            testtools.TestCase.skipException = unittest.case.SkipTest
+        else:
+            testtools.TestCase.skipException = unittest2.case.SkipTest
+    except Exception:
+        pass
+
+
 class BaseTestCase(testtools.testcase.WithAttributes, testtools.TestCase):
     setUpClassCalled = False
 
@@ -33,6 +51,18 @@ class BaseTestCase(testtools.testcase.WithAttributes, testtools.TestCase):
         if hasattr(super(BaseTestCase, cls), 'setUpClass'):
             super(BaseTestCase, cls).setUpClass()
         cls.setUpClassCalled = True
+        # TODO(gmann): cls.handle_skip_exception is really workaround for
+        # testtools bug- https://github.com/testing-cabal/testtools/issues/272
+        # stestr which is used by Tempest internally to run the test switch
+        # the customize test runner(which use stdlib unittest) for >=py3.5
+        # else testtools.run.- https://github.com/mtreinish/stestr/pull/265
+        # These two test runner are not compatible due to skip exception
+        # handling(due to unittest2). testtools.run treat unittestt.SkipTest
+        # as error and stdlib unittest treat unittest2.case.SkipTest raised
+        # by testtools.TestCase.skipException.
+        # The below workaround can be removed once testtools fix issue# 272.
+        cls.orig_skip_exception = testtools.TestCase.skipException
+        _handle_skip_exception()
 
     @classmethod
     def tearDownClass(cls):
@@ -40,6 +70,7 @@ class BaseTestCase(testtools.testcase.WithAttributes, testtools.TestCase):
             super(BaseTestCase, cls).tearDownClass()
 
     def setUp(self):
+        testtools.TestCase.skipException = self.orig_skip_exception
         super(BaseTestCase, self).setUp()
         if not self.setUpClassCalled:
             raise RuntimeError("setUpClass does not calls the super's "

tempest/test.py

@@ -20,7 +20,6 @@ import sys
 import debtcollector.moves
 import fixtures
 from oslo_log import log as logging
-import pkg_resources
 import six
 import testtools
 
@@ -28,6 +27,7 @@ from tempest import clients
 from tempest.common import credentials_factory as credentials
 from tempest.common import utils
 from tempest import config
+from tempest.lib import base as lib_base
 from tempest.lib.common import fixed_network
 from tempest.lib.common import profiler
 from tempest.lib.common import validation_resources as vr
@@ -78,10 +78,6 @@ def validate_tearDownClass():
 atexit.register(validate_tearDownClass)
 
 
-class DummyException(Exception):
-    pass
-
-
 class BaseTestCase(testtools.testcase.WithAttributes,
                    testtools.TestCase):
     """The test base class defines Tempest framework for class level fixtures.
@@ -144,26 +140,6 @@ class BaseTestCase(testtools.testcase.WithAttributes,
         # Stack of (name, callable) to be invoked in reverse order at teardown
         cls._teardowns = []
 
-    @classmethod
-    def handle_skip_exception(cls):
-        try:
-            stestr_version = pkg_resources.parse_version(
-                pkg_resources.get_distribution("stestr").version)
-            stestr_min = pkg_resources.parse_version('2.5.0')
-            new_stestr = (stestr_version >= stestr_min)
-            import unittest
-            import unittest2
-            if sys.version_info >= (3, 5) and new_stestr:
-                exc = unittest2.case.SkipTest
-                exc_to_raise = unittest.case.SkipTest
-            else:
-                exc = unittest.case.SkipTest
-                exc_to_raise = unittest2.case.SkipTest
-        except Exception:
-            exc = DummyException
-            exc_to_raise = DummyException
-        return exc, exc_to_raise
-
     @classmethod
     def setUpClass(cls):
         cls.__setupclass_called = True
@@ -183,8 +159,9 @@ class BaseTestCase(testtools.testcase.WithAttributes,
         # as error and stdlib unittest treat unittest2.case.SkipTest raised
         # by testtools.TestCase.skipException.
         # The below workaround can be removed once testtools fix issue# 272.
+        orig_skip_exception = testtools.TestCase.skipException
+        lib_base._handle_skip_exception()
         try:
-            exc, exc_to_raise = cls.handle_skip_exception()
             cls.skip_checks()
 
             if not cls.__skip_checks_called:
@@ -202,12 +179,6 @@ class BaseTestCase(testtools.testcase.WithAttributes,
             # Additional class-wide test resources
             cls._teardowns.append(('resources', cls.resource_cleanup))
             cls.resource_setup()
-        except exc as e:
-            # NOTE(dviroel): the exception may be raised after setting up the
-            # user credentials, so we must call tearDownClass to release all
-            # allocated resources.
-            cls.tearDownClass()
-            raise exc_to_raise(e.args)
         except Exception:
             etype, value, trace = sys.exc_info()
             LOG.info("%s raised in %s.setUpClass. Invoking tearDownClass.",
@@ -217,6 +188,8 @@ class BaseTestCase(testtools.testcase.WithAttributes,
                 six.reraise(etype, value, trace)
             finally:
                 del trace  # to avoid circular refs
+        finally:
+            testtools.TestCase.skipException = orig_skip_exception
 
     @classmethod
     def tearDownClass(cls):

tempest/tests/lib/test_base.py

@@ -48,6 +48,7 @@ class TestSetUpClass(base.BaseTestCase):
     @classmethod
     def setUpClass(cls):  # noqa
         """Simulate absence of super() call."""
+        cls.orig_skip_exception = cls.skipException
 
     def setUp(self):
         try:

tempest/tests/test_test.py

@@ -531,8 +531,8 @@ class TestTempestBaseTestClassFixtures(base.TestCase):
     def test_skip_only(self):
         # If a skip condition is hit in the test, no credentials or resource
         # is provisioned / cleaned-up
-        exc, _ = test.BaseTestCase.handle_skip_exception()
-        self.mocks['skip_checks'].side_effect = (exc)
+        self.mocks['skip_checks'].side_effect = (
+            testtools.TestCase.skipException())
         suite = unittest.TestSuite((self.test,))
         log = []
         result = LoggingTestResult(log)