Merge "Framework for staged setup"
This commit is contained in:
commit
07a3f45625
42
HACKING.rst
42
HACKING.rst
@ -11,7 +11,7 @@ Tempest Specific Commandments
|
|||||||
- [T102] Cannot import OpenStack python clients in tempest/api &
|
- [T102] Cannot import OpenStack python clients in tempest/api &
|
||||||
tempest/scenario tests
|
tempest/scenario tests
|
||||||
- [T104] Scenario tests require a services decorator
|
- [T104] Scenario tests require a services decorator
|
||||||
- [T105] Unit tests cannot use setUpClass
|
- [T105] Tests cannot use setUpClass/tearDownClass
|
||||||
- [T106] vim configuration should not be kept in source files.
|
- [T106] vim configuration should not be kept in source files.
|
||||||
- [N322] Method's default argument shouldn't be mutable
|
- [N322] Method's default argument shouldn't be mutable
|
||||||
|
|
||||||
@ -108,6 +108,46 @@ name. For example, any test that make an api call to a service other than nova
|
|||||||
in tempest.api.compute would require a service tag for those services, however
|
in tempest.api.compute would require a service tag for those services, however
|
||||||
they do not need to be tagged as compute.
|
they do not need to be tagged as compute.
|
||||||
|
|
||||||
|
Test fixtures and resources
|
||||||
|
---------------------------
|
||||||
|
Test level resources should be cleaned-up after the test execution. Clean-up
|
||||||
|
is best scheduled using `addCleanup` which ensures that the resource cleanup
|
||||||
|
code is always invoked, and in reverse order with respect to the creation
|
||||||
|
order.
|
||||||
|
|
||||||
|
Test class level resources should be defined in the `resource_setup` method of
|
||||||
|
the test class, except for any credential obtained from the credentials
|
||||||
|
provider, which should be set-up in the `setup_credentials` method.
|
||||||
|
|
||||||
|
The test base class `BaseTestCase` defines Tempest framework for class level
|
||||||
|
fixtures. `setUpClass` and `tearDownClass` are defined here and cannot be
|
||||||
|
overwritten by subclasses (enforced via hacking rule T105).
|
||||||
|
|
||||||
|
Set-up is split in a series of steps (setup stages), which can be overwritten
|
||||||
|
by test classes. Set-up stages are:
|
||||||
|
- `skip_checks`
|
||||||
|
- `setup_credentials`
|
||||||
|
- `setup_clients`
|
||||||
|
- `resource_setup`
|
||||||
|
|
||||||
|
Tear-down is also split in a series of steps (teardown stages), which are
|
||||||
|
stacked for execution only if the corresponding setup stage had been
|
||||||
|
reached during the setup phase. Tear-down stages are:
|
||||||
|
- `clear_isolated_creds` (defined in the base test class)
|
||||||
|
- `resource_cleanup`
|
||||||
|
|
||||||
|
Skipping Tests
|
||||||
|
--------------
|
||||||
|
Skipping tests should be based on configuration only. If that is not possible,
|
||||||
|
it is likely that either a configuration flag is missing, or the test should
|
||||||
|
fail rather than be skipped.
|
||||||
|
Using discovery for skipping tests is generally discouraged.
|
||||||
|
|
||||||
|
When running a test that requires a certain "feature" in the target
|
||||||
|
cloud, if that feature is missing we should fail, because either the test
|
||||||
|
configuration is invalid, or the cloud is broken and the expected "feature" is
|
||||||
|
not there even if the cloud was configured with it.
|
||||||
|
|
||||||
Negative Tests
|
Negative Tests
|
||||||
--------------
|
--------------
|
||||||
Newly added negative tests should use the negative test framework. First step
|
Newly added negative tests should use the negative test framework. First step
|
||||||
|
117
tempest/test.py
117
tempest/test.py
@ -224,6 +224,23 @@ atexit.register(validate_tearDownClass)
|
|||||||
|
|
||||||
class BaseTestCase(testtools.testcase.WithAttributes,
|
class BaseTestCase(testtools.testcase.WithAttributes,
|
||||||
testtools.TestCase):
|
testtools.TestCase):
|
||||||
|
"""The test base class defines Tempest framework for class level fixtures.
|
||||||
|
`setUpClass` and `tearDownClass` are defined here and cannot be overwritten
|
||||||
|
by subclasses (enforced via hacking rule T105).
|
||||||
|
|
||||||
|
Set-up is split in a series of steps (setup stages), which can be
|
||||||
|
overwritten by test classes. Set-up stages are:
|
||||||
|
- skip_checks
|
||||||
|
- setup_credentials
|
||||||
|
- setup_clients
|
||||||
|
- resource_setup
|
||||||
|
|
||||||
|
Tear-down is also split in a series of steps (teardown stages), which are
|
||||||
|
stacked for execution only if the corresponding setup stage had been
|
||||||
|
reached during the setup phase. Tear-down stages are:
|
||||||
|
- clear_isolated_creds (defined in the base test class)
|
||||||
|
- resource_cleanup
|
||||||
|
"""
|
||||||
|
|
||||||
setUpClassCalled = False
|
setUpClassCalled = False
|
||||||
_service = None
|
_service = None
|
||||||
@ -242,31 +259,28 @@ class BaseTestCase(testtools.testcase.WithAttributes,
|
|||||||
if hasattr(super(BaseTestCase, cls), 'setUpClass'):
|
if hasattr(super(BaseTestCase, cls), 'setUpClass'):
|
||||||
super(BaseTestCase, cls).setUpClass()
|
super(BaseTestCase, cls).setUpClass()
|
||||||
cls.setUpClassCalled = True
|
cls.setUpClassCalled = True
|
||||||
# No test resource is allocated until here
|
# Stack of (name, callable) to be invoked in reverse order at teardown
|
||||||
|
cls.teardowns = []
|
||||||
|
# All the configuration checks that may generate a skip
|
||||||
|
cls.skip_checks()
|
||||||
try:
|
try:
|
||||||
# TODO(andreaf) Split-up resource_setup in stages:
|
# Allocation of all required credentials and client managers
|
||||||
# skip checks, pre-hook, credentials, clients, resources, post-hook
|
cls.teardowns.append(('credentials', cls.clear_isolated_creds))
|
||||||
|
cls.setup_credentials()
|
||||||
|
# Shortcuts to clients
|
||||||
|
cls.setup_clients()
|
||||||
|
# Additional class-wide test resources
|
||||||
|
cls.teardowns.append(('resources', cls.resource_cleanup))
|
||||||
cls.resource_setup()
|
cls.resource_setup()
|
||||||
except Exception:
|
except Exception:
|
||||||
etype, value, trace = sys.exc_info()
|
etype, value, trace = sys.exc_info()
|
||||||
LOG.info("%s in resource setup. Invoking tearDownClass." % etype)
|
LOG.info("%s in %s.setUpClass. Invoking tearDownClass." % (
|
||||||
# Catch any exception in tearDown so we can re-raise the original
|
cls.__name__, etype))
|
||||||
# exception at the end
|
|
||||||
try:
|
|
||||||
cls.tearDownClass()
|
cls.tearDownClass()
|
||||||
except Exception as te:
|
|
||||||
tetype, _, _ = sys.exc_info()
|
|
||||||
# TODO(gmann): Till we split-up resource_setup &
|
|
||||||
# resource_cleanup in more structural way, log
|
|
||||||
# AttributeError as info instead of exception.
|
|
||||||
if tetype is AttributeError:
|
|
||||||
LOG.info("tearDownClass failed: %s" % te)
|
|
||||||
else:
|
|
||||||
LOG.exception("tearDownClass failed: %s" % te)
|
|
||||||
try:
|
try:
|
||||||
raise etype(value), None, trace
|
raise etype, value, trace
|
||||||
finally:
|
finally:
|
||||||
del trace # for avoiding circular refs
|
del trace # to avoid circular refs
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def tearDownClass(cls):
|
def tearDownClass(cls):
|
||||||
@ -274,21 +288,78 @@ class BaseTestCase(testtools.testcase.WithAttributes,
|
|||||||
# It should never be overridden by descendants
|
# It should never be overridden by descendants
|
||||||
if hasattr(super(BaseTestCase, cls), 'tearDownClass'):
|
if hasattr(super(BaseTestCase, cls), 'tearDownClass'):
|
||||||
super(BaseTestCase, cls).tearDownClass()
|
super(BaseTestCase, cls).tearDownClass()
|
||||||
|
# Save any existing exception, we always want to re-raise the original
|
||||||
|
# exception only
|
||||||
|
etype, value, trace = sys.exc_info()
|
||||||
|
# If there was no exception during setup we shall re-raise the first
|
||||||
|
# exception in teardown
|
||||||
|
re_raise = (etype is None)
|
||||||
|
while cls.teardowns:
|
||||||
|
name, teardown = cls.teardowns.pop()
|
||||||
|
# Catch any exception in tearDown so we can re-raise the original
|
||||||
|
# exception at the end
|
||||||
try:
|
try:
|
||||||
cls.resource_cleanup()
|
teardown()
|
||||||
|
except Exception as te:
|
||||||
|
sys_exec_info = sys.exc_info()
|
||||||
|
tetype = sys_exec_info[0]
|
||||||
|
# TODO(andreaf): Till we have the ability to cleanup only
|
||||||
|
# resources that were successfully setup in resource_cleanup,
|
||||||
|
# log AttributeError as info instead of exception.
|
||||||
|
if tetype is AttributeError and name == 'resources':
|
||||||
|
LOG.info("tearDownClass of %s failed: %s" % (name, te))
|
||||||
|
else:
|
||||||
|
LOG.exception("teardown of %s failed: %s" % (name, te))
|
||||||
|
if not etype:
|
||||||
|
etype, value, trace = sys_exec_info
|
||||||
|
# If exceptions were raised during teardown, an not before, re-raise
|
||||||
|
# the first one
|
||||||
|
if re_raise and etype is not None:
|
||||||
|
try:
|
||||||
|
raise etype, value, trace
|
||||||
finally:
|
finally:
|
||||||
cls.clear_isolated_creds()
|
del trace # to avoid circular refs
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def resource_setup(cls):
|
def resource_setup(cls):
|
||||||
"""Class level setup steps for test cases.
|
"""Class level resource setup for test cases.
|
||||||
Recommended order: skip checks, credentials, clients, resources.
|
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def resource_cleanup(cls):
|
def resource_cleanup(cls):
|
||||||
"""Class level resource cleanup for test cases. """
|
"""Class level resource cleanup for test cases.
|
||||||
|
Resource cleanup must be able to handle the case of partially setup
|
||||||
|
resources, in case a failure during `resource_setup` should happen.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def skip_checks(cls):
|
||||||
|
"""Class level skip checks. Subclasses verify in here all
|
||||||
|
conditions that might prevent the execution of the entire test class.
|
||||||
|
Checks implemented here may not make use API calls, and should rely on
|
||||||
|
configuration alone.
|
||||||
|
In general skip checks that require an API call are discouraged.
|
||||||
|
If one is really needed it may be implemented either in the
|
||||||
|
resource_setup or at test level.
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setup_credentials(cls):
|
||||||
|
"""Allocate credentials and the client managers from them."""
|
||||||
|
# TODO(andreaf) There is a fair amount of code that could me moved from
|
||||||
|
# base / test classes in here. Ideally tests should be able to only
|
||||||
|
# specify a list of (additional) credentials the need to use.
|
||||||
|
pass
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def setup_clients(cls):
|
||||||
|
"""Create links to the clients into the test object."""
|
||||||
|
# TODO(andreaf) There is a fair amount of code that could me moved from
|
||||||
|
# base / test classes in here. Ideally tests should be able to only
|
||||||
|
# specify which client is `client` and nothing else.
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
|
Loading…
Reference in New Issue
Block a user