Add workaround to handle the testtool skip exception in CLI test

This commit apply the workaround for testtool skip exception
in CLI test base class.
- https://review.opendev.org/#/c/681340/6

I have modified the workaround done in above commit to more generic
one.

We are good on skipException from test code. That is handled by
teststools.TestCase's self.exception_handlers
- f51ce5f934/testtools/testcase.py (L275)
Mapped handler _report_skip() will add the respective tests to skip list

Current CLI test failure
-https://zuul.opendev.org/t/openstack/build/c1bb61121e1d4d108c52adad200e8991/log/job-output.txt#6354

I have tested both test.BaseTestCase and lib.base.BaseTestCase with all 8 combinations
of below:
 - py >= 3.5 and py < 3.5 (on py2.7)
 - stestr >= 2.50 and stestr < 2.5.0
 - skip exception from setUpClass and skip exception from test code

Closes-Bug: 1847749

Change-Id: Ib70bdffaf4d38743e7cbbeb88af51bb57ceeedf6
changes/99/687999/8
Ghanshyam Mann 3 years ago
parent f1f1cd943c
commit e64c78dcf7
  1. 6
      releasenotes/notes/fix-1847749-2670b1d4f6097a1a.yaml
  2. 31
      tempest/lib/base.py
  3. 37
      tempest/test.py
  4. 1
      tempest/tests/lib/test_base.py
  5. 4
      tempest/tests/test_test.py

@ -0,0 +1,6 @@
---
fixes:
- |
Bug#1847749. This privides the workaround of Skip Exception raised instead of skipping
the CLI tests. If you are running Tempest with stestr > 2.5.0 then use this fix.
Ref- https://github.com/testing-cabal/testtools/issues/272

@ -14,11 +14,29 @@
# under the License.
import os
import sys
import fixtures
import pkg_resources
import testtools
def _handle_skip_exception():
try:
stestr_version = pkg_resources.parse_version(
pkg_resources.get_distribution("stestr").version)
stestr_min = pkg_resources.parse_version('2.5.0')
new_stestr = (stestr_version >= stestr_min)
import unittest
import unittest2
if sys.version_info >= (3, 5) and new_stestr:
testtools.TestCase.skipException = unittest.case.SkipTest
else:
testtools.TestCase.skipException = unittest2.case.SkipTest
except Exception:
pass
class BaseTestCase(testtools.testcase.WithAttributes, testtools.TestCase):
setUpClassCalled = False
@ -33,6 +51,18 @@ class BaseTestCase(testtools.testcase.WithAttributes, testtools.TestCase):
if hasattr(super(BaseTestCase, cls), 'setUpClass'):
super(BaseTestCase, cls).setUpClass()
cls.setUpClassCalled = True
# TODO(gmann): cls.handle_skip_exception is really workaround for
# testtools bug- https://github.com/testing-cabal/testtools/issues/272
# stestr which is used by Tempest internally to run the test switch
# the customize test runner(which use stdlib unittest) for >=py3.5
# else testtools.run.- https://github.com/mtreinish/stestr/pull/265
# These two test runner are not compatible due to skip exception
# handling(due to unittest2). testtools.run treat unittestt.SkipTest
# as error and stdlib unittest treat unittest2.case.SkipTest raised
# by testtools.TestCase.skipException.
# The below workaround can be removed once testtools fix issue# 272.
cls.orig_skip_exception = testtools.TestCase.skipException
_handle_skip_exception()
@classmethod
def tearDownClass(cls):
@ -40,6 +70,7 @@ class BaseTestCase(testtools.testcase.WithAttributes, testtools.TestCase):
super(BaseTestCase, cls).tearDownClass()
def setUp(self):
testtools.TestCase.skipException = self.orig_skip_exception
super(BaseTestCase, self).setUp()
if not self.setUpClassCalled:
raise RuntimeError("setUpClass does not calls the super's "

@ -20,7 +20,6 @@ import sys
import debtcollector.moves
import fixtures
from oslo_log import log as logging
import pkg_resources
import six
import testtools
@ -28,6 +27,7 @@ from tempest import clients
from tempest.common import credentials_factory as credentials
from tempest.common import utils
from tempest import config
from tempest.lib import base as lib_base
from tempest.lib.common import fixed_network
from tempest.lib.common import profiler
from tempest.lib.common import validation_resources as vr
@ -78,10 +78,6 @@ def validate_tearDownClass():
atexit.register(validate_tearDownClass)
class DummyException(Exception):
pass
class BaseTestCase(testtools.testcase.WithAttributes,
testtools.TestCase):
"""The test base class defines Tempest framework for class level fixtures.
@ -144,26 +140,6 @@ class BaseTestCase(testtools.testcase.WithAttributes,
# Stack of (name, callable) to be invoked in reverse order at teardown
cls._teardowns = []
@classmethod
def handle_skip_exception(cls):
try:
stestr_version = pkg_resources.parse_version(
pkg_resources.get_distribution("stestr").version)
stestr_min = pkg_resources.parse_version('2.5.0')
new_stestr = (stestr_version >= stestr_min)
import unittest
import unittest2
if sys.version_info >= (3, 5) and new_stestr:
exc = unittest2.case.SkipTest
exc_to_raise = unittest.case.SkipTest
else:
exc = unittest.case.SkipTest
exc_to_raise = unittest2.case.SkipTest
except Exception:
exc = DummyException
exc_to_raise = DummyException
return exc, exc_to_raise
@classmethod
def setUpClass(cls):
cls.__setupclass_called = True
@ -183,8 +159,9 @@ class BaseTestCase(testtools.testcase.WithAttributes,
# as error and stdlib unittest treat unittest2.case.SkipTest raised
# by testtools.TestCase.skipException.
# The below workaround can be removed once testtools fix issue# 272.
orig_skip_exception = testtools.TestCase.skipException
lib_base._handle_skip_exception()
try:
exc, exc_to_raise = cls.handle_skip_exception()
cls.skip_checks()
if not cls.__skip_checks_called:
@ -202,12 +179,6 @@ class BaseTestCase(testtools.testcase.WithAttributes,
# Additional class-wide test resources
cls._teardowns.append(('resources', cls.resource_cleanup))
cls.resource_setup()
except exc as e:
# NOTE(dviroel): the exception may be raised after setting up the
# user credentials, so we must call tearDownClass to release all
# allocated resources.
cls.tearDownClass()
raise exc_to_raise(e.args)
except Exception:
etype, value, trace = sys.exc_info()
LOG.info("%s raised in %s.setUpClass. Invoking tearDownClass.",
@ -217,6 +188,8 @@ class BaseTestCase(testtools.testcase.WithAttributes,
six.reraise(etype, value, trace)
finally:
del trace # to avoid circular refs
finally:
testtools.TestCase.skipException = orig_skip_exception
@classmethod
def tearDownClass(cls):

@ -48,6 +48,7 @@ class TestSetUpClass(base.BaseTestCase):
@classmethod
def setUpClass(cls): # noqa
"""Simulate absence of super() call."""
cls.orig_skip_exception = cls.skipException
def setUp(self):
try:

@ -531,8 +531,8 @@ class TestTempestBaseTestClassFixtures(base.TestCase):
def test_skip_only(self):
# If a skip condition is hit in the test, no credentials or resource
# is provisioned / cleaned-up
exc, _ = test.BaseTestCase.handle_skip_exception()
self.mocks['skip_checks'].side_effect = (exc)
self.mocks['skip_checks'].side_effect = (
testtools.TestCase.skipException())
suite = unittest.TestSuite((self.test,))
log = []
result = LoggingTestResult(log)

Loading…
Cancel
Save