diff --git a/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml new file mode 100644 index 0000000000..2203fd1a30 --- /dev/null +++ b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + New decorator ``unstable_test`` is added to ``tempest.lib.decorators``. + It can be used to mark some test as unstable thus it will be still run + by tempest but job will not fail if this test will fail. Such test will + be skipped in case of failure. + It can be used for example when there is known bug related which cause + irregular tests failures. Marking such test as unstable will help other + developers to get their job done and still run this test to get additional + debug data or to confirm if some potential fix really solved the issue. diff --git a/tempest/lib/decorators.py b/tempest/lib/decorators.py index b399aa0519..a716fa23fc 100644 --- a/tempest/lib/decorators.py +++ b/tempest/lib/decorators.py @@ -147,3 +147,45 @@ def attr(**kwargs): return f return decorator + + +def unstable_test(*args, **kwargs): + """A decorator useful to run tests hitting known bugs and skip it if fails + + This decorator can be used in cases like: + + * We have skipped tests with some bug and now bug is claimed to be fixed. + Now we want to check the test stability so we use this decorator. + The number of skipped cases with that bug can be counted to mark test + stable again. + * There is test which is failing often, but not always. If there is known + bug related to it, and someone is working on fix, this decorator can be + used instead of "skip_because". That will ensure that test is still run + so new debug data can be collected from jobs' logs but it will not make + life of other developers harder by forcing them to recheck jobs more + often. + + ``bug`` must be a number for the test to skip. + + :param bug: bug number causing the test to skip (launchpad or storyboard) + :param bug_type: 'launchpad' or 'storyboard', default 'launchpad' + :raises: testtools.TestCase.skipException if test actually fails, + and ``bug`` is included + """ + def decor(f): + @functools.wraps(f) + def inner(self, *func_args, **func_kwargs): + try: + return f(self, *func_args, **func_kwargs) + except Exception as e: + if "bug" in kwargs: + bug = kwargs['bug'] + bug_type = kwargs.get('bug_type', 'launchpad') + bug_url = _get_bug_url(bug, bug_type) + msg = ("Marked as unstable and skipped because of bug: " + "%s, failure was: %s") % (bug_url, e) + raise testtools.TestCase.skipException(msg) + else: + raise e + return inner + return decor diff --git a/tempest/tests/lib/test_decorators.py b/tempest/tests/lib/test_decorators.py index 0b1a5991d5..59c4e2d996 100644 --- a/tempest/tests/lib/test_decorators.py +++ b/tempest/tests/lib/test_decorators.py @@ -13,7 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +import abc + import mock +import six import testtools from tempest.lib import base as test @@ -51,9 +54,36 @@ class TestAttrDecorator(base.TestCase): self._test_attr_helper(expected_attrs=['foo'], type=['foo', 'foo']) -class TestSkipBecauseDecorator(base.TestCase): - def _test_skip_because_helper(self, expected_to_skip=True, - **decorator_args): +@six.add_metaclass(abc.ABCMeta) +class BaseSkipDecoratorTests(object): + + @abc.abstractmethod + def _test_skip_helper(self, raise_exception=True, expected_to_skip=True, + **decorator_args): + return + + def test_skip_launchpad_bug(self): + self._test_skip_helper(bug='12345') + + def test_skip_storyboard_bug(self): + self._test_skip_helper(bug='1992', bug_type='storyboard') + + def test_skip_bug_without_bug_never_skips(self): + """Never skip without a bug parameter.""" + self._test_skip_helper( + raise_exception=False, expected_to_skip=False, condition=True) + self._test_skip_helper( + raise_exception=False, expected_to_skip=False) + + def test_skip_invalid_bug_number(self): + """Raise InvalidParam if with an invalid bug number""" + self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper, + bug='critical_bug') + + +class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests): + def _test_skip_helper(self, raise_exception=True, expected_to_skip=True, + **decorator_args): class TestFoo(test.BaseTestCase): _interface = 'json' @@ -75,38 +105,56 @@ class TestSkipBecauseDecorator(base.TestCase): # assert that test_bar returned 0 self.assertEqual(TestFoo('test_bar').test_bar(), 0) - def test_skip_because_launchpad_bug(self): - self._test_skip_because_helper(bug='12345') - def test_skip_because_launchpad_bug_and_condition_true(self): - self._test_skip_because_helper(bug='12348', condition=True) + self._test_skip_helper(bug='12348', condition=True) def test_skip_because_launchpad_bug_and_condition_false(self): - self._test_skip_because_helper(expected_to_skip=False, - bug='12349', condition=False) - - def test_skip_because_storyboard_bug(self): - self._test_skip_because_helper(bug='1992', bug_type='storyboard') - - def test_skip_because_storyboard_bug_and_condition_true(self): - self._test_skip_because_helper(bug='1992', bug_type='storyboard', - condition=True) + self._test_skip_helper(expected_to_skip=False, + bug='12349', condition=False) def test_skip_because_storyboard_bug_and_condition_false(self): - self._test_skip_because_helper(expected_to_skip=False, - bug='1992', bug_type='storyboard', - condition=False) + self._test_skip_helper(expected_to_skip=False, + bug='1992', bug_type='storyboard', + condition=False) - def test_skip_because_bug_without_bug_never_skips(self): - """Never skip without a bug parameter.""" - self._test_skip_because_helper(expected_to_skip=False, - condition=True) - self._test_skip_because_helper(expected_to_skip=False) + def test_skip_because_storyboard_bug_and_condition_true(self): + self._test_skip_helper(bug='1992', bug_type='storyboard', + condition=True) - def test_skip_because_invalid_bug_number(self): - """Raise InvalidParam if with an invalid bug number""" - self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper, - bug='critical_bug') + +class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests): + + def _test_skip_helper(self, raise_exception=True, expected_to_skip=True, + **decorator_args): + fail_test_reason = "test_bar failed" + + class TestFoo(test.BaseTestCase): + + @decorators.unstable_test(**decorator_args) + def test_bar(self): + if raise_exception: + raise Exception(fail_test_reason) + else: + return 0 + + t = TestFoo('test_bar') + if expected_to_skip: + e = self.assertRaises(testtools.TestCase.skipException, t.test_bar) + bug = decorator_args['bug'] + bug_type = decorator_args.get('bug_type', 'launchpad') + self.assertRegex( + str(e), + r'Marked as unstable and skipped because of bug\: %s.*, ' + 'failure was: %s' % (decorators._get_bug_url(bug, bug_type), + fail_test_reason) + ) + else: + # assert that test_bar returned 0 + self.assertEqual(TestFoo('test_bar').test_bar(), 0) + + def test_skip_bug_given_exception_not_raised(self): + self._test_skip_helper(raise_exception=False, expected_to_skip=False, + bug='1234') class TestIdempotentIdDecorator(base.TestCase):