From 21f53012f76d11e3df327adcf87e67edf9045d09 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Tue, 29 Jan 2019 12:52:01 +0100 Subject: [PATCH] Add unstable_test decorator This decorator can be used to temporarily mark some tests as unstable. This may help sometimes to debug such test which is failing often but not always in the gate because it will still be run but will not cause all job failure when it fails. This may be also used to easily track in logstash how often such test is failing by looking for describption of unstability reason set in decorator. Change-Id: I79ce70f479506ec2b3216747d533ff2e450685aa Related-Bug: #1813198 --- ...table_test-decorator-a73cf97d4ffcc796.yaml | 11 ++ tempest/lib/decorators.py | 42 +++++++ tempest/tests/lib/test_decorators.py | 104 +++++++++++++----- 3 files changed, 129 insertions(+), 28 deletions(-) create mode 100644 releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml diff --git a/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml new file mode 100644 index 0000000000..2203fd1a30 --- /dev/null +++ b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + New decorator ``unstable_test`` is added to ``tempest.lib.decorators``. + It can be used to mark some test as unstable thus it will be still run + by tempest but job will not fail if this test will fail. Such test will + be skipped in case of failure. + It can be used for example when there is known bug related which cause + irregular tests failures. Marking such test as unstable will help other + developers to get their job done and still run this test to get additional + debug data or to confirm if some potential fix really solved the issue. diff --git a/tempest/lib/decorators.py b/tempest/lib/decorators.py index b399aa0519..a716fa23fc 100644 --- a/tempest/lib/decorators.py +++ b/tempest/lib/decorators.py @@ -147,3 +147,45 @@ def attr(**kwargs): return f return decorator + + +def unstable_test(*args, **kwargs): + """A decorator useful to run tests hitting known bugs and skip it if fails + + This decorator can be used in cases like: + + * We have skipped tests with some bug and now bug is claimed to be fixed. + Now we want to check the test stability so we use this decorator. + The number of skipped cases with that bug can be counted to mark test + stable again. + * There is test which is failing often, but not always. If there is known + bug related to it, and someone is working on fix, this decorator can be + used instead of "skip_because". That will ensure that test is still run + so new debug data can be collected from jobs' logs but it will not make + life of other developers harder by forcing them to recheck jobs more + often. + + ``bug`` must be a number for the test to skip. + + :param bug: bug number causing the test to skip (launchpad or storyboard) + :param bug_type: 'launchpad' or 'storyboard', default 'launchpad' + :raises: testtools.TestCase.skipException if test actually fails, + and ``bug`` is included + """ + def decor(f): + @functools.wraps(f) + def inner(self, *func_args, **func_kwargs): + try: + return f(self, *func_args, **func_kwargs) + except Exception as e: + if "bug" in kwargs: + bug = kwargs['bug'] + bug_type = kwargs.get('bug_type', 'launchpad') + bug_url = _get_bug_url(bug, bug_type) + msg = ("Marked as unstable and skipped because of bug: " + "%s, failure was: %s") % (bug_url, e) + raise testtools.TestCase.skipException(msg) + else: + raise e + return inner + return decor diff --git a/tempest/tests/lib/test_decorators.py b/tempest/tests/lib/test_decorators.py index 0b1a5991d5..59c4e2d996 100644 --- a/tempest/tests/lib/test_decorators.py +++ b/tempest/tests/lib/test_decorators.py @@ -13,7 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +import abc + import mock +import six import testtools from tempest.lib import base as test @@ -51,9 +54,36 @@ class TestAttrDecorator(base.TestCase): self._test_attr_helper(expected_attrs=['foo'], type=['foo', 'foo']) -class TestSkipBecauseDecorator(base.TestCase): - def _test_skip_because_helper(self, expected_to_skip=True, - **decorator_args): +@six.add_metaclass(abc.ABCMeta) +class BaseSkipDecoratorTests(object): + + @abc.abstractmethod + def _test_skip_helper(self, raise_exception=True, expected_to_skip=True, + **decorator_args): + return + + def test_skip_launchpad_bug(self): + self._test_skip_helper(bug='12345') + + def test_skip_storyboard_bug(self): + self._test_skip_helper(bug='1992', bug_type='storyboard') + + def test_skip_bug_without_bug_never_skips(self): + """Never skip without a bug parameter.""" + self._test_skip_helper( + raise_exception=False, expected_to_skip=False, condition=True) + self._test_skip_helper( + raise_exception=False, expected_to_skip=False) + + def test_skip_invalid_bug_number(self): + """Raise InvalidParam if with an invalid bug number""" + self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper, + bug='critical_bug') + + +class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests): + def _test_skip_helper(self, raise_exception=True, expected_to_skip=True, + **decorator_args): class TestFoo(test.BaseTestCase): _interface = 'json' @@ -75,38 +105,56 @@ class TestSkipBecauseDecorator(base.TestCase): # assert that test_bar returned 0 self.assertEqual(TestFoo('test_bar').test_bar(), 0) - def test_skip_because_launchpad_bug(self): - self._test_skip_because_helper(bug='12345') - def test_skip_because_launchpad_bug_and_condition_true(self): - self._test_skip_because_helper(bug='12348', condition=True) + self._test_skip_helper(bug='12348', condition=True) def test_skip_because_launchpad_bug_and_condition_false(self): - self._test_skip_because_helper(expected_to_skip=False, - bug='12349', condition=False) - - def test_skip_because_storyboard_bug(self): - self._test_skip_because_helper(bug='1992', bug_type='storyboard') - - def test_skip_because_storyboard_bug_and_condition_true(self): - self._test_skip_because_helper(bug='1992', bug_type='storyboard', - condition=True) + self._test_skip_helper(expected_to_skip=False, + bug='12349', condition=False) def test_skip_because_storyboard_bug_and_condition_false(self): - self._test_skip_because_helper(expected_to_skip=False, - bug='1992', bug_type='storyboard', - condition=False) + self._test_skip_helper(expected_to_skip=False, + bug='1992', bug_type='storyboard', + condition=False) - def test_skip_because_bug_without_bug_never_skips(self): - """Never skip without a bug parameter.""" - self._test_skip_because_helper(expected_to_skip=False, - condition=True) - self._test_skip_because_helper(expected_to_skip=False) + def test_skip_because_storyboard_bug_and_condition_true(self): + self._test_skip_helper(bug='1992', bug_type='storyboard', + condition=True) - def test_skip_because_invalid_bug_number(self): - """Raise InvalidParam if with an invalid bug number""" - self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper, - bug='critical_bug') + +class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests): + + def _test_skip_helper(self, raise_exception=True, expected_to_skip=True, + **decorator_args): + fail_test_reason = "test_bar failed" + + class TestFoo(test.BaseTestCase): + + @decorators.unstable_test(**decorator_args) + def test_bar(self): + if raise_exception: + raise Exception(fail_test_reason) + else: + return 0 + + t = TestFoo('test_bar') + if expected_to_skip: + e = self.assertRaises(testtools.TestCase.skipException, t.test_bar) + bug = decorator_args['bug'] + bug_type = decorator_args.get('bug_type', 'launchpad') + self.assertRegex( + str(e), + r'Marked as unstable and skipped because of bug\: %s.*, ' + 'failure was: %s' % (decorators._get_bug_url(bug, bug_type), + fail_test_reason) + ) + else: + # assert that test_bar returned 0 + self.assertEqual(TestFoo('test_bar').test_bar(), 0) + + def test_skip_bug_given_exception_not_raised(self): + self._test_skip_helper(raise_exception=False, expected_to_skip=False, + bug='1234') class TestIdempotentIdDecorator(base.TestCase):