Merge "Add unstable_test decorator"
This commit is contained in:
Zuul
Gerrit Code Review
commit
e6057592fb
@ -0,0 +1,11 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
New decorator ``unstable_test`` is added to ``tempest.lib.decorators``.
|
||||
It can be used to mark some test as unstable thus it will be still run
|
||||
by tempest but job will not fail if this test will fail. Such test will
|
||||
be skipped in case of failure.
|
||||
It can be used for example when there is known bug related which cause
|
||||
irregular tests failures. Marking such test as unstable will help other
|
||||
developers to get their job done and still run this test to get additional
|
||||
debug data or to confirm if some potential fix really solved the issue.
|
||||
@ -154,3 +154,45 @@ def attr(**kwargs):
|
||||
return f
|
||||
|
||||
return decorator
|
||||
|
||||
|
||||
def unstable_test(*args, **kwargs):
|
||||
"""A decorator useful to run tests hitting known bugs and skip it if fails
|
||||
|
||||
This decorator can be used in cases like:
|
||||
|
||||
* We have skipped tests with some bug and now bug is claimed to be fixed.
|
||||
Now we want to check the test stability so we use this decorator.
|
||||
The number of skipped cases with that bug can be counted to mark test
|
||||
stable again.
|
||||
* There is test which is failing often, but not always. If there is known
|
||||
bug related to it, and someone is working on fix, this decorator can be
|
||||
used instead of "skip_because". That will ensure that test is still run
|
||||
so new debug data can be collected from jobs' logs but it will not make
|
||||
life of other developers harder by forcing them to recheck jobs more
|
||||
often.
|
||||
|
||||
``bug`` must be a number for the test to skip.
|
||||
|
||||
:param bug: bug number causing the test to skip (launchpad or storyboard)
|
||||
:param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
|
||||
:raises: testtools.TestCase.skipException if test actually fails,
|
||||
and ``bug`` is included
|
||||
"""
|
||||
def decor(f):
|
||||
@functools.wraps(f)
|
||||
def inner(self, *func_args, **func_kwargs):
|
||||
try:
|
||||
return f(self, *func_args, **func_kwargs)
|
||||
except Exception as e:
|
||||
if "bug" in kwargs:
|
||||
bug = kwargs['bug']
|
||||
bug_type = kwargs.get('bug_type', 'launchpad')
|
||||
bug_url = _get_bug_url(bug, bug_type)
|
||||
msg = ("Marked as unstable and skipped because of bug: "
|
||||
"%s, failure was: %s") % (bug_url, e)
|
||||
raise testtools.TestCase.skipException(msg)
|
||||
else:
|
||||
raise e
|
||||
return inner
|
||||
return decor
|
||||
|
||||
@ -13,7 +13,10 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
|
||||
import mock
|
||||
import six
|
||||
import testtools
|
||||
|
||||
from tempest.lib import base as test
|
||||
@ -66,9 +69,36 @@ class TestAttrDecorator(base.TestCase):
|
||||
condition=True)
|
||||
|
||||
|
||||
class TestSkipBecauseDecorator(base.TestCase):
|
||||
def _test_skip_because_helper(self, expected_to_skip=True,
|
||||
**decorator_args):
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class BaseSkipDecoratorTests(object):
|
||||
|
||||
@abc.abstractmethod
|
||||
def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
|
||||
**decorator_args):
|
||||
return
|
||||
|
||||
def test_skip_launchpad_bug(self):
|
||||
self._test_skip_helper(bug='12345')
|
||||
|
||||
def test_skip_storyboard_bug(self):
|
||||
self._test_skip_helper(bug='1992', bug_type='storyboard')
|
||||
|
||||
def test_skip_bug_without_bug_never_skips(self):
|
||||
"""Never skip without a bug parameter."""
|
||||
self._test_skip_helper(
|
||||
raise_exception=False, expected_to_skip=False, condition=True)
|
||||
self._test_skip_helper(
|
||||
raise_exception=False, expected_to_skip=False)
|
||||
|
||||
def test_skip_invalid_bug_number(self):
|
||||
"""Raise InvalidParam if with an invalid bug number"""
|
||||
self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper,
|
||||
bug='critical_bug')
|
||||
|
||||
|
||||
class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests):
|
||||
def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
|
||||
**decorator_args):
|
||||
class TestFoo(test.BaseTestCase):
|
||||
_interface = 'json'
|
||||
|
||||
@ -90,38 +120,56 @@ class TestSkipBecauseDecorator(base.TestCase):
|
||||
# assert that test_bar returned 0
|
||||
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
|
||||
|
||||
def test_skip_because_launchpad_bug(self):
|
||||
self._test_skip_because_helper(bug='12345')
|
||||
|
||||
def test_skip_because_launchpad_bug_and_condition_true(self):
|
||||
self._test_skip_because_helper(bug='12348', condition=True)
|
||||
self._test_skip_helper(bug='12348', condition=True)
|
||||
|
||||
def test_skip_because_launchpad_bug_and_condition_false(self):
|
||||
self._test_skip_because_helper(expected_to_skip=False,
|
||||
bug='12349', condition=False)
|
||||
|
||||
def test_skip_because_storyboard_bug(self):
|
||||
self._test_skip_because_helper(bug='1992', bug_type='storyboard')
|
||||
|
||||
def test_skip_because_storyboard_bug_and_condition_true(self):
|
||||
self._test_skip_because_helper(bug='1992', bug_type='storyboard',
|
||||
condition=True)
|
||||
self._test_skip_helper(expected_to_skip=False,
|
||||
bug='12349', condition=False)
|
||||
|
||||
def test_skip_because_storyboard_bug_and_condition_false(self):
|
||||
self._test_skip_because_helper(expected_to_skip=False,
|
||||
bug='1992', bug_type='storyboard',
|
||||
condition=False)
|
||||
self._test_skip_helper(expected_to_skip=False,
|
||||
bug='1992', bug_type='storyboard',
|
||||
condition=False)
|
||||
|
||||
def test_skip_because_bug_without_bug_never_skips(self):
|
||||
"""Never skip without a bug parameter."""
|
||||
self._test_skip_because_helper(expected_to_skip=False,
|
||||
condition=True)
|
||||
self._test_skip_because_helper(expected_to_skip=False)
|
||||
def test_skip_because_storyboard_bug_and_condition_true(self):
|
||||
self._test_skip_helper(bug='1992', bug_type='storyboard',
|
||||
condition=True)
|
||||
|
||||
def test_skip_because_invalid_bug_number(self):
|
||||
"""Raise InvalidParam if with an invalid bug number"""
|
||||
self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper,
|
||||
bug='critical_bug')
|
||||
|
||||
class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests):
|
||||
|
||||
def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
|
||||
**decorator_args):
|
||||
fail_test_reason = "test_bar failed"
|
||||
|
||||
class TestFoo(test.BaseTestCase):
|
||||
|
||||
@decorators.unstable_test(**decorator_args)
|
||||
def test_bar(self):
|
||||
if raise_exception:
|
||||
raise Exception(fail_test_reason)
|
||||
else:
|
||||
return 0
|
||||
|
||||
t = TestFoo('test_bar')
|
||||
if expected_to_skip:
|
||||
e = self.assertRaises(testtools.TestCase.skipException, t.test_bar)
|
||||
bug = decorator_args['bug']
|
||||
bug_type = decorator_args.get('bug_type', 'launchpad')
|
||||
self.assertRegex(
|
||||
str(e),
|
||||
r'Marked as unstable and skipped because of bug\: %s.*, '
|
||||
'failure was: %s' % (decorators._get_bug_url(bug, bug_type),
|
||||
fail_test_reason)
|
||||
)
|
||||
else:
|
||||
# assert that test_bar returned 0
|
||||
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
|
||||
|
||||
def test_skip_bug_given_exception_not_raised(self):
|
||||
self._test_skip_helper(raise_exception=False, expected_to_skip=False,
|
||||
bug='1234')
|
||||
|
||||
|
||||
class TestIdempotentIdDecorator(base.TestCase):
|
||||
|
||||
Loading…
Reference in New Issue
Block a user