Merge "Add unstable_test decorator"
This commit is contained in:
commit
e6057592fb
|
@ -0,0 +1,11 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
New decorator ``unstable_test`` is added to ``tempest.lib.decorators``.
|
||||||
|
It can be used to mark some test as unstable thus it will be still run
|
||||||
|
by tempest but job will not fail if this test will fail. Such test will
|
||||||
|
be skipped in case of failure.
|
||||||
|
It can be used for example when there is known bug related which cause
|
||||||
|
irregular tests failures. Marking such test as unstable will help other
|
||||||
|
developers to get their job done and still run this test to get additional
|
||||||
|
debug data or to confirm if some potential fix really solved the issue.
|
|
@ -154,3 +154,45 @@ def attr(**kwargs):
|
||||||
return f
|
return f
|
||||||
|
|
||||||
return decorator
|
return decorator
|
||||||
|
|
||||||
|
|
||||||
|
def unstable_test(*args, **kwargs):
|
||||||
|
"""A decorator useful to run tests hitting known bugs and skip it if fails
|
||||||
|
|
||||||
|
This decorator can be used in cases like:
|
||||||
|
|
||||||
|
* We have skipped tests with some bug and now bug is claimed to be fixed.
|
||||||
|
Now we want to check the test stability so we use this decorator.
|
||||||
|
The number of skipped cases with that bug can be counted to mark test
|
||||||
|
stable again.
|
||||||
|
* There is test which is failing often, but not always. If there is known
|
||||||
|
bug related to it, and someone is working on fix, this decorator can be
|
||||||
|
used instead of "skip_because". That will ensure that test is still run
|
||||||
|
so new debug data can be collected from jobs' logs but it will not make
|
||||||
|
life of other developers harder by forcing them to recheck jobs more
|
||||||
|
often.
|
||||||
|
|
||||||
|
``bug`` must be a number for the test to skip.
|
||||||
|
|
||||||
|
:param bug: bug number causing the test to skip (launchpad or storyboard)
|
||||||
|
:param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
|
||||||
|
:raises: testtools.TestCase.skipException if test actually fails,
|
||||||
|
and ``bug`` is included
|
||||||
|
"""
|
||||||
|
def decor(f):
|
||||||
|
@functools.wraps(f)
|
||||||
|
def inner(self, *func_args, **func_kwargs):
|
||||||
|
try:
|
||||||
|
return f(self, *func_args, **func_kwargs)
|
||||||
|
except Exception as e:
|
||||||
|
if "bug" in kwargs:
|
||||||
|
bug = kwargs['bug']
|
||||||
|
bug_type = kwargs.get('bug_type', 'launchpad')
|
||||||
|
bug_url = _get_bug_url(bug, bug_type)
|
||||||
|
msg = ("Marked as unstable and skipped because of bug: "
|
||||||
|
"%s, failure was: %s") % (bug_url, e)
|
||||||
|
raise testtools.TestCase.skipException(msg)
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
return inner
|
||||||
|
return decor
|
||||||
|
|
|
@ -13,7 +13,10 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
import abc
|
||||||
|
|
||||||
import mock
|
import mock
|
||||||
|
import six
|
||||||
import testtools
|
import testtools
|
||||||
|
|
||||||
from tempest.lib import base as test
|
from tempest.lib import base as test
|
||||||
|
@ -66,9 +69,36 @@ class TestAttrDecorator(base.TestCase):
|
||||||
condition=True)
|
condition=True)
|
||||||
|
|
||||||
|
|
||||||
class TestSkipBecauseDecorator(base.TestCase):
|
@six.add_metaclass(abc.ABCMeta)
|
||||||
def _test_skip_because_helper(self, expected_to_skip=True,
|
class BaseSkipDecoratorTests(object):
|
||||||
**decorator_args):
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
|
||||||
|
**decorator_args):
|
||||||
|
return
|
||||||
|
|
||||||
|
def test_skip_launchpad_bug(self):
|
||||||
|
self._test_skip_helper(bug='12345')
|
||||||
|
|
||||||
|
def test_skip_storyboard_bug(self):
|
||||||
|
self._test_skip_helper(bug='1992', bug_type='storyboard')
|
||||||
|
|
||||||
|
def test_skip_bug_without_bug_never_skips(self):
|
||||||
|
"""Never skip without a bug parameter."""
|
||||||
|
self._test_skip_helper(
|
||||||
|
raise_exception=False, expected_to_skip=False, condition=True)
|
||||||
|
self._test_skip_helper(
|
||||||
|
raise_exception=False, expected_to_skip=False)
|
||||||
|
|
||||||
|
def test_skip_invalid_bug_number(self):
|
||||||
|
"""Raise InvalidParam if with an invalid bug number"""
|
||||||
|
self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper,
|
||||||
|
bug='critical_bug')
|
||||||
|
|
||||||
|
|
||||||
|
class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests):
|
||||||
|
def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
|
||||||
|
**decorator_args):
|
||||||
class TestFoo(test.BaseTestCase):
|
class TestFoo(test.BaseTestCase):
|
||||||
_interface = 'json'
|
_interface = 'json'
|
||||||
|
|
||||||
|
@ -90,38 +120,56 @@ class TestSkipBecauseDecorator(base.TestCase):
|
||||||
# assert that test_bar returned 0
|
# assert that test_bar returned 0
|
||||||
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
|
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
|
||||||
|
|
||||||
def test_skip_because_launchpad_bug(self):
|
|
||||||
self._test_skip_because_helper(bug='12345')
|
|
||||||
|
|
||||||
def test_skip_because_launchpad_bug_and_condition_true(self):
|
def test_skip_because_launchpad_bug_and_condition_true(self):
|
||||||
self._test_skip_because_helper(bug='12348', condition=True)
|
self._test_skip_helper(bug='12348', condition=True)
|
||||||
|
|
||||||
def test_skip_because_launchpad_bug_and_condition_false(self):
|
def test_skip_because_launchpad_bug_and_condition_false(self):
|
||||||
self._test_skip_because_helper(expected_to_skip=False,
|
self._test_skip_helper(expected_to_skip=False,
|
||||||
bug='12349', condition=False)
|
bug='12349', condition=False)
|
||||||
|
|
||||||
def test_skip_because_storyboard_bug(self):
|
|
||||||
self._test_skip_because_helper(bug='1992', bug_type='storyboard')
|
|
||||||
|
|
||||||
def test_skip_because_storyboard_bug_and_condition_true(self):
|
|
||||||
self._test_skip_because_helper(bug='1992', bug_type='storyboard',
|
|
||||||
condition=True)
|
|
||||||
|
|
||||||
def test_skip_because_storyboard_bug_and_condition_false(self):
|
def test_skip_because_storyboard_bug_and_condition_false(self):
|
||||||
self._test_skip_because_helper(expected_to_skip=False,
|
self._test_skip_helper(expected_to_skip=False,
|
||||||
bug='1992', bug_type='storyboard',
|
bug='1992', bug_type='storyboard',
|
||||||
condition=False)
|
condition=False)
|
||||||
|
|
||||||
def test_skip_because_bug_without_bug_never_skips(self):
|
def test_skip_because_storyboard_bug_and_condition_true(self):
|
||||||
"""Never skip without a bug parameter."""
|
self._test_skip_helper(bug='1992', bug_type='storyboard',
|
||||||
self._test_skip_because_helper(expected_to_skip=False,
|
condition=True)
|
||||||
condition=True)
|
|
||||||
self._test_skip_because_helper(expected_to_skip=False)
|
|
||||||
|
|
||||||
def test_skip_because_invalid_bug_number(self):
|
|
||||||
"""Raise InvalidParam if with an invalid bug number"""
|
class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests):
|
||||||
self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper,
|
|
||||||
bug='critical_bug')
|
def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
|
||||||
|
**decorator_args):
|
||||||
|
fail_test_reason = "test_bar failed"
|
||||||
|
|
||||||
|
class TestFoo(test.BaseTestCase):
|
||||||
|
|
||||||
|
@decorators.unstable_test(**decorator_args)
|
||||||
|
def test_bar(self):
|
||||||
|
if raise_exception:
|
||||||
|
raise Exception(fail_test_reason)
|
||||||
|
else:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
t = TestFoo('test_bar')
|
||||||
|
if expected_to_skip:
|
||||||
|
e = self.assertRaises(testtools.TestCase.skipException, t.test_bar)
|
||||||
|
bug = decorator_args['bug']
|
||||||
|
bug_type = decorator_args.get('bug_type', 'launchpad')
|
||||||
|
self.assertRegex(
|
||||||
|
str(e),
|
||||||
|
r'Marked as unstable and skipped because of bug\: %s.*, '
|
||||||
|
'failure was: %s' % (decorators._get_bug_url(bug, bug_type),
|
||||||
|
fail_test_reason)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# assert that test_bar returned 0
|
||||||
|
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
|
||||||
|
|
||||||
|
def test_skip_bug_given_exception_not_raised(self):
|
||||||
|
self._test_skip_helper(raise_exception=False, expected_to_skip=False,
|
||||||
|
bug='1234')
|
||||||
|
|
||||||
|
|
||||||
class TestIdempotentIdDecorator(base.TestCase):
|
class TestIdempotentIdDecorator(base.TestCase):
|
||||||
|
|
Loading…
Reference in New Issue