Merge "Add unstable_test decorator"

releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    New decorator ``unstable_test`` is added to ``tempest.lib.decorators``.
+    It can be used to mark some test as unstable thus it will be still run
+    by tempest but job will not fail if this test will fail. Such test will
+    be skipped in case of failure.
+    It can be used for example when there is known bug related which cause
+    irregular tests failures. Marking such test as unstable will help other
+    developers to get their job done and still run this test to get additional
+    debug data or to confirm if some potential fix really solved the issue.

tempest/lib/decorators.py

@@ -154,3 +154,45 @@ def attr(**kwargs):
         return f
 
     return decorator
+
+
+def unstable_test(*args, **kwargs):
+    """A decorator useful to run tests hitting known bugs and skip it if fails
+
+    This decorator can be used in cases like:
+
+    * We have skipped tests with some bug and now bug is claimed to be fixed.
+      Now we want to check the test stability so we use this decorator.
+      The number of skipped cases with that bug can be counted to mark test
+      stable again.
+    * There is test which is failing often, but not always. If there is known
+      bug related to it, and someone is working on fix, this decorator can be
+      used instead of "skip_because". That will ensure that test is still run
+      so new debug data can be collected from jobs' logs but it will not make
+      life of other developers harder by forcing them to recheck jobs more
+      often.
+
+    ``bug`` must be a number for the test to skip.
+
+    :param bug: bug number causing the test to skip (launchpad or storyboard)
+    :param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
+    :raises: testtools.TestCase.skipException if test actually fails,
+        and ``bug`` is included
+    """
+    def decor(f):
+        @functools.wraps(f)
+        def inner(self, *func_args, **func_kwargs):
+            try:
+                return f(self, *func_args, **func_kwargs)
+            except Exception as e:
+                if "bug" in kwargs:
+                    bug = kwargs['bug']
+                    bug_type = kwargs.get('bug_type', 'launchpad')
+                    bug_url = _get_bug_url(bug, bug_type)
+                    msg = ("Marked as unstable and skipped because of bug: "
+                           "%s, failure was: %s") % (bug_url, e)
+                    raise testtools.TestCase.skipException(msg)
+                else:
+                    raise e
+        return inner
+    return decor

tempest/tests/lib/test_decorators.py

@@ -13,7 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import abc
+
 import mock
+import six
 import testtools
 
 from tempest.lib import base as test
@@ -66,9 +69,36 @@ class TestAttrDecorator(base.TestCase):
                                condition=True)
 
 
-class TestSkipBecauseDecorator(base.TestCase):
-    def _test_skip_because_helper(self, expected_to_skip=True,
-                                  **decorator_args):
+@six.add_metaclass(abc.ABCMeta)
+class BaseSkipDecoratorTests(object):
+
+    @abc.abstractmethod
+    def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+                          **decorator_args):
+        return
+
+    def test_skip_launchpad_bug(self):
+        self._test_skip_helper(bug='12345')
+
+    def test_skip_storyboard_bug(self):
+        self._test_skip_helper(bug='1992', bug_type='storyboard')
+
+    def test_skip_bug_without_bug_never_skips(self):
+        """Never skip without a bug parameter."""
+        self._test_skip_helper(
+            raise_exception=False, expected_to_skip=False, condition=True)
+        self._test_skip_helper(
+            raise_exception=False, expected_to_skip=False)
+
+    def test_skip_invalid_bug_number(self):
+        """Raise InvalidParam if with an invalid bug number"""
+        self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper,
+                          bug='critical_bug')
+
+
+class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests):
+    def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+                          **decorator_args):
         class TestFoo(test.BaseTestCase):
             _interface = 'json'
 
@@ -90,38 +120,56 @@ class TestSkipBecauseDecorator(base.TestCase):
             # assert that test_bar returned 0
             self.assertEqual(TestFoo('test_bar').test_bar(), 0)
 
-    def test_skip_because_launchpad_bug(self):
-        self._test_skip_because_helper(bug='12345')
-
     def test_skip_because_launchpad_bug_and_condition_true(self):
-        self._test_skip_because_helper(bug='12348', condition=True)
+        self._test_skip_helper(bug='12348', condition=True)
 
     def test_skip_because_launchpad_bug_and_condition_false(self):
-        self._test_skip_because_helper(expected_to_skip=False,
-                                       bug='12349', condition=False)
+        self._test_skip_helper(expected_to_skip=False,
+                               bug='12349', condition=False)
 
-    def test_skip_because_storyboard_bug(self):
-        self._test_skip_because_helper(bug='1992', bug_type='storyboard')
+    def test_skip_because_storyboard_bug_and_condition_false(self):
+        self._test_skip_helper(expected_to_skip=False,
+                               bug='1992', bug_type='storyboard',
+                               condition=False)
 
     def test_skip_because_storyboard_bug_and_condition_true(self):
-        self._test_skip_because_helper(bug='1992', bug_type='storyboard',
-                                       condition=True)
+        self._test_skip_helper(bug='1992', bug_type='storyboard',
+                               condition=True)
 
-    def test_skip_because_storyboard_bug_and_condition_false(self):
-        self._test_skip_because_helper(expected_to_skip=False,
-                                       bug='1992', bug_type='storyboard',
-                                       condition=False)
 
-    def test_skip_because_bug_without_bug_never_skips(self):
-        """Never skip without a bug parameter."""
-        self._test_skip_because_helper(expected_to_skip=False,
-                                       condition=True)
-        self._test_skip_because_helper(expected_to_skip=False)
+class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests):
 
-    def test_skip_because_invalid_bug_number(self):
-        """Raise InvalidParam if with an invalid bug number"""
-        self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper,
-                          bug='critical_bug')
+    def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+                          **decorator_args):
+        fail_test_reason = "test_bar failed"
+
+        class TestFoo(test.BaseTestCase):
+
+            @decorators.unstable_test(**decorator_args)
+            def test_bar(self):
+                if raise_exception:
+                    raise Exception(fail_test_reason)
+                else:
+                    return 0
+
+        t = TestFoo('test_bar')
+        if expected_to_skip:
+            e = self.assertRaises(testtools.TestCase.skipException, t.test_bar)
+            bug = decorator_args['bug']
+            bug_type = decorator_args.get('bug_type', 'launchpad')
+            self.assertRegex(
+                str(e),
+                r'Marked as unstable and skipped because of bug\: %s.*, '
+                'failure was: %s' % (decorators._get_bug_url(bug, bug_type),
+                                     fail_test_reason)
+            )
+        else:
+            # assert that test_bar returned 0
+            self.assertEqual(TestFoo('test_bar').test_bar(), 0)
+
+    def test_skip_bug_given_exception_not_raised(self):
+        self._test_skip_helper(raise_exception=False, expected_to_skip=False,
+                               bug='1234')
 
 
 class TestIdempotentIdDecorator(base.TestCase):