openstack/tempest
Code Issues Proposed changes

Merge "Added storyboard integration to tempest.lib decorators"

Browse Source
This commit is contained in:
Zuul 2018-07-27 00:42:06 +00:00 committed by Gerrit Code Review
commit fc8ef3aeeb
3 changed files with 160 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
releasenotes/notes/add-storyboard-in-skip-because-decorator-3e139aa8a4f7970f.yaml Normal file
View File

@ -0,0 +1,17 @@
---
features:
- |
Add a new parameter called ``bug_type`` to
``tempest.lib.decorators.related_bug`` and
``tempest.lib.decorators.skip_because`` decorators, which accepts
2 values:
* launchpad
* storyboard
This offers the possibility of tracking bugs related to tests using
launchpad or storyboard references. The default value is launchpad
for backward compatibility.
Passing in a non-digit ``bug`` value to either decorator will raise
a ``InvalidParam`` exception (previously ``ValueError``).

69
tempest/lib/decorators.py
View File

@ -19,39 +19,83 @@ from oslo_log import log as logging
import six
import testtools
from tempest.lib import exceptions as lib_exc
LOG = logging.getLogger(__name__)
_SUPPORTED_BUG_TYPES = {
'launchpad': 'https://launchpad.net/bugs/%s',
'storyboard': 'https://storyboard.openstack.org/#!/story/%s',
}
def _validate_bug_and_bug_type(bug, bug_type):
"""Validates ``bug`` and ``bug_type`` values.
:param bug: bug number causing the test to skip (launchpad or storyboard)
:param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
:raises: InvalidParam if ``bug`` is not a digit or ``bug_type`` is not
a valid value
"""
if not bug.isdigit():
invalid_param = '%s must be a valid %s number' % (bug, bug_type)
raise lib_exc.InvalidParam(invalid_param=invalid_param)
if bug_type not in _SUPPORTED_BUG_TYPES:
invalid_param = 'bug_type "%s" must be one of: %s' % (
bug_type, ', '.join(_SUPPORTED_BUG_TYPES.keys()))
raise lib_exc.InvalidParam(invalid_param=invalid_param)
def _get_bug_url(bug, bug_type='launchpad'):
"""Get the bug URL based on the ``bug_type`` and ``bug``
:param bug: The launchpad/storyboard bug number causing the test
:param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
:returns: Bug URL corresponding to ``bug_type`` value
"""
_validate_bug_and_bug_type(bug, bug_type)
return _SUPPORTED_BUG_TYPES[bug_type] % bug
def skip_because(*args, **kwargs):
"""A decorator useful to skip tests hitting known bugs
@param bug: bug number causing the test to skip
@param condition: optional condition to be True for the skip to have place
``bug`` must be a number and ``condition`` must be true for the test to
skip.
:param bug: bug number causing the test to skip (launchpad or storyboard)
:param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
:param condition: optional condition to be True for the skip to have place
:raises: testtools.TestCase.skipException if ``condition`` is True and
``bug`` is included
"""
def decorator(f):
@functools.wraps(f)
def wrapper(*func_args, **func_kwargs):
skip = False
msg = ''
if "condition" in kwargs:
if kwargs["condition"] is True:
skip = True
else:
skip = True
if "bug" in kwargs and skip is True:
if not kwargs['bug'].isdigit():
raise ValueError('bug must be a valid bug number')
msg = "Skipped until Bug: %s is resolved." % kwargs["bug"]
bug = kwargs['bug']
bug_type = kwargs.get('bug_type', 'launchpad')
bug_url = _get_bug_url(bug, bug_type)
msg = "Skipped until bug: %s is resolved." % bug_url
raise testtools.TestCase.skipException(msg)
return f(*func_args, **func_kwargs)
return wrapper
return decorator
def related_bug(bug, status_code=None):
"""A decorator useful to know solutions from launchpad bug reports
def related_bug(bug, status_code=None, bug_type='launchpad'):
"""A decorator useful to know solutions from launchpad/storyboard reports
@param bug: The launchpad bug number causing the test
@param status_code: The status code related to the bug report
:param bug: The launchpad/storyboard bug number causing the test bug
:param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
:param status_code: The status code related to the bug report
"""
def decorator(f):
@functools.wraps(f)
@ -61,9 +105,10 @@ def related_bug(bug, status_code=None):
except Exception as exc:
exc_status_code = getattr(exc, 'status_code', None)
if status_code is None or status_code == exc_status_code:
LOG.error('Hints: This test was made for the bug %s. '
'The failure could be related to '
'https://launchpad.net/bugs/%s', bug, bug)
if bug:
LOG.error('Hints: This test was made for the bug_type '
'%s. The failure could be related to '
'%s', bug, _get_bug_url(bug, bug_type))
raise exc
return wrapper
return decorator

97
tempest/tests/lib/test_decorators.py
View File

@ -19,6 +19,7 @@ import testtools
from tempest.lib import base as test
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
from tempest.lib import exceptions as lib_exc
from tempest.tests import base
@ -62,21 +63,40 @@ class TestSkipBecauseDecorator(base.TestCase):
t = TestFoo('test_bar')
if expected_to_skip:
self.assertRaises(testtools.TestCase.skipException, t.test_bar)
e = self.assertRaises(testtools.TestCase.skipException, t.test_bar)
bug = decorator_args['bug']
bug_type = decorator_args.get('bug_type', 'launchpad')
self.assertRegex(
str(e),
r'Skipped until bug\: %s.*' % decorators._get_bug_url(
bug, bug_type)
)
else:
# assert that test_bar returned 0
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
def test_skip_because_bug(self):
def test_skip_because_launchpad_bug(self):
self._test_skip_because_helper(bug='12345')
def test_skip_because_bug_and_condition_true(self):
def test_skip_because_launchpad_bug_and_condition_true(self):
self._test_skip_because_helper(bug='12348', condition=True)
def test_skip_because_bug_and_condition_false(self):
def test_skip_because_launchpad_bug_and_condition_false(self):
self._test_skip_because_helper(expected_to_skip=False,
bug='12349', condition=False)
def test_skip_because_storyboard_bug(self):
self._test_skip_because_helper(bug='1992', bug_type='storyboard')
def test_skip_because_storyboard_bug_and_condition_true(self):
self._test_skip_because_helper(bug='1992', bug_type='storyboard',
condition=True)
def test_skip_because_storyboard_bug_and_condition_false(self):
self._test_skip_because_helper(expected_to_skip=False,
bug='1992', bug_type='storyboard',
condition=False)
def test_skip_because_bug_without_bug_never_skips(self):
"""Never skip without a bug parameter."""
self._test_skip_because_helper(expected_to_skip=False,
@ -84,8 +104,8 @@ class TestSkipBecauseDecorator(base.TestCase):
self._test_skip_because_helper(expected_to_skip=False)
def test_skip_because_invalid_bug_number(self):
"""Raise ValueError if with an invalid bug number"""
self.assertRaises(ValueError, self._test_skip_because_helper,
"""Raise InvalidParam if with an invalid bug number"""
self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper,
bug='critical_bug')
@ -126,6 +146,13 @@ class TestIdempotentIdDecorator(base.TestCase):
class TestRelatedBugDecorator(base.TestCase):
def _get_my_exception(self):
class MyException(Exception):
def __init__(self, status_code):
self.status_code = status_code
return MyException
def test_relatedbug_when_no_exception(self):
f = mock.Mock()
sentinel = object()
@ -137,10 +164,9 @@ class TestRelatedBugDecorator(base.TestCase):
test_foo(sentinel)
f.assert_called_once_with(sentinel)
def test_relatedbug_when_exception(self):
class MyException(Exception):
def __init__(self, status_code):
self.status_code = status_code
def test_relatedbug_when_exception_with_launchpad_bug_type(self):
"""Validate related_bug decorator with bug_type == 'launchpad'"""
MyException = self._get_my_exception()
def f(self):
raise MyException(status_code=500)
@ -152,4 +178,53 @@ class TestRelatedBugDecorator(base.TestCase):
with mock.patch.object(decorators.LOG, 'error') as m_error:
self.assertRaises(MyException, test_foo, object())
m_error.assert_called_once_with(mock.ANY, '1234', '1234')
m_error.assert_called_once_with(
mock.ANY, '1234', 'https://launchpad.net/bugs/1234')
def test_relatedbug_when_exception_with_storyboard_bug_type(self):
"""Validate related_bug decorator with bug_type == 'storyboard'"""
MyException = self._get_my_exception()
def f(self):
raise MyException(status_code=500)
@decorators.related_bug(bug="1234", status_code=500,
bug_type='storyboard')
def test_foo(self):
f(self)
with mock.patch.object(decorators.LOG, 'error') as m_error:
self.assertRaises(MyException, test_foo, object())
m_error.assert_called_once_with(
mock.ANY, '1234', 'https://storyboard.openstack.org/#!/story/1234')
def test_relatedbug_when_exception_invalid_bug_type(self):
"""Check related_bug decorator raises exc when bug_type is not valid"""
MyException = self._get_my_exception()
def f(self):
raise MyException(status_code=500)
@decorators.related_bug(bug="1234", status_code=500,
bug_type=mock.sentinel.invalid)
def test_foo(self):
f(self)
with mock.patch.object(decorators.LOG, 'error'):
self.assertRaises(lib_exc.InvalidParam, test_foo, object())
def test_relatedbug_when_exception_invalid_bug_number(self):
"""Check related_bug decorator raises exc when bug_number != digit"""
MyException = self._get_my_exception()
def f(self):
raise MyException(status_code=500)
@decorators.related_bug(bug="not a digit", status_code=500,
bug_type='launchpad')
def test_foo(self):
f(self)
with mock.patch.object(decorators.LOG, 'error'):
self.assertRaises(lib_exc.InvalidParam, test_foo, object())