Add breakpoint support

This covers most of the features specified in:

http://specs.openstack.org/openstack/heat-specs/specs/juno/stack-breakpoint.html

The breakpoints are specified via hooks in the stack's environment.

The only thing missing from the blueprint is stepping through a stack.

Partial-Blueprint: stack-breakpoint
Change-Id: Iddc019464484af18ca6f21f11660649e30d63aca
This commit is contained in:
Tomas Sedovic 2015-03-18 23:54:19 -04:00 committed by Steven Hardy
parent 073a9d3404
commit f5e428a0bb
7 changed files with 563 additions and 6 deletions

View File

@ -129,3 +129,55 @@ resource. The supported URL types are "http, https and file".
resources:
my_db_server:
"OS::DBInstance": file:///home/mine/all_my_cool_templates/db.yaml
7) Pause stack creation/update on a given resource
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
If you want to debug your stack as it's being created or updated or if you want
to run it in phases you can set `pre-create` and `pre-update` hooks in the
`resources` section of `resource_registry`.
To set a hook, add either `hooks: pre-create` or `hooks: pre-update` to the
resource's dictionary. You can also use the `[pre-create, pre-update]` to stop
on both actions.
Hooks can be combined with other `resources` properties (e.g. provider
templates or type mapping).
Example:
::
resource_registry:
resources:
my_server:
"OS::DBInstance": file:///home/mine/all_my_cool_templates/db.yaml
hooks: pre-create
nested_stack:
nested_resource:
hooks: pre-update
another_resource:
hooks: [pre-create, pre-update]
When Heat encounters a resource that has a hook, it will pause the resource
action until the hook is cleared. Any resources that depend on it will wait as
well. Any resources that don't will be created in parallel (unless they have
hooks, too).
It is also possible to do a partial match by putting an asterisk (`*`) in the
name.
This example:
::
resource_registry:
resources:
"*_server":
hooks: pre-create
will pause while creating `app_server` and `database_server` but not `server`
or `app_network`.
Hook is cleared by signalling the resource with `{unset_hook: pre-create}` (or
`pre-update`).

View File

@ -11,7 +11,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import copy
import fnmatch
import glob
import itertools
import os.path
@ -32,6 +34,24 @@ from heat.engine import support
LOG = log.getLogger(__name__)
HOOK_TYPES = (HOOK_PRE_CREATE, HOOK_PRE_UPDATE) = ('pre-create', 'pre-update')
def valid_hook_type(hook):
return hook in HOOK_TYPES
def is_hook_definition(key, value):
if key == 'hooks':
if isinstance(value, six.string_types):
return valid_hook_type(value)
elif isinstance(value, collections.Sequence):
return all(valid_hook_type(hook) for hook in value)
else:
return False
return False
class ResourceInfo(object):
"""Base mapping of resource type to implementation."""
@ -174,12 +194,23 @@ class ResourceRegistry(object):
for k, v in iter(registry.items()):
if v is None:
self._register_info(path + [k], None)
elif is_hook_definition(k, v):
self._register_hook(path + [k], v)
elif isinstance(v, dict):
self._load_registry(path + [k], v)
else:
self._register_info(path + [k],
ResourceInfo(self, path + [k], v))
def _register_hook(self, path, hook):
name = path[-1]
registry = self._registry
for key in path[:-1]:
if key not in registry:
registry[key] = {}
registry = registry[key]
registry[name] = hook
def _register_info(self, path, info):
"""place the new info in the correct location in the registry.
path: a list of keys ['resources', 'my_server', 'OS::Nova::Server']
@ -242,6 +273,53 @@ class ResourceRegistry(object):
if info.path[-1] in registry:
registry.pop(info.path[-1])
def matches_hook(self, resource_name, hook):
'''Return whether a resource have a hook set in the environment.
For a given resource and a hook type, we check to see if the the passed
group of resources has the right hook associated with the name.
Hooks are set in this format via `resources`:
{
"res_name": {
"hooks": [pre-create, pre-update]
},
"*_suffix": {
"hooks": pre-create
},
"prefix_*": {
"hooks": pre-update
}
}
A hook value is either `pre-create`, `pre-update` or a list of those
values. Resources support wildcard matching. The asterisk sign matches
everything.
'''
ress = self._registry['resources']
for name_pattern, resource in six.iteritems(ress):
if fnmatch.fnmatchcase(resource_name, name_pattern):
if 'hooks' in resource:
hooks = resource['hooks']
if isinstance(hooks, six.string_types):
if hook == hooks:
return True
elif isinstance(hooks, collections.Sequence):
if hook in hooks:
return True
return False
def remove_resources_except(self, resource_name):
ress = self._registry['resources']
new_resources = {}
for name, res in six.iteritems(ress):
if fnmatch.fnmatchcase(resource_name, name):
new_resources.update(res)
if resource_name in ress:
new_resources.update(ress[resource_name])
self._registry['resources'] = new_resources
def iterable_by(self, resource_type, resource_name=None):
is_templ_type = resource_type.endswith(('.yaml', '.template'))
if self.global_registry is not None and is_templ_type:
@ -334,6 +412,8 @@ class ResourceRegistry(object):
for k, v in iter(level.items()):
if isinstance(v, dict):
tmp[k] = _as_dict(v)
elif is_hook_definition(k, v):
tmp[k] = v
elif v.user_resource:
tmp[k] = v.value
return tmp
@ -445,7 +525,8 @@ class Environment(object):
return self.stack_lifecycle_plugins
def get_child_environment(parent_env, child_params, item_to_remove=None):
def get_child_environment(parent_env, child_params, item_to_remove=None,
child_resource_name=None):
"""Build a child environment using the parent environment and params.
This is built from the child_params and the parent env so some
@ -456,6 +537,10 @@ def get_child_environment(parent_env, child_params, item_to_remove=None):
parent env to take presdence).
2. child parameters must overwrite the parent's as they won't be relevant
in the child template.
If `child_resource_name` is provided, resources in the registry will be
replaced with the contents of the matching child resource plus anything
that passes a wildcard match.
"""
def is_flat_params(env_or_param):
if env_or_param is None:
@ -478,6 +563,9 @@ def get_child_environment(parent_env, child_params, item_to_remove=None):
if item_to_remove is not None:
new_env.registry.remove_item(item_to_remove)
if child_resource_name:
new_env.registry.remove_resources_except(child_resource_name)
return new_env

View File

@ -31,6 +31,7 @@ from heat.common import identifier
from heat.common import short_id
from heat.common import timeutils
from heat.engine import attributes
from heat.engine import environment
from heat.engine import event
from heat.engine import function
from heat.engine import properties
@ -265,6 +266,34 @@ class Resource(object):
rs.update_and_save({'rsrc_metadata': metadata})
self._rsrc_metadata = metadata
def _break_if_required(self, action, hook):
'''Block the resource until the hook is cleared if there is one.'''
if self.stack.env.registry.matches_hook(self.name, hook):
self._add_event(self.action, self.status,
_("%(a)s paused until Hook %(h)s is cleared")
% {'a': action, 'h': hook})
self.trigger_hook(hook)
LOG.info(_LI('Reached hook on %s'), six.text_type(self))
while self.has_hook(hook) and self.status != self.FAILED:
try:
yield
except Exception:
self.clear_hook(hook)
self._add_event(
self.action, self.status,
"Failure occured while waiting.")
def has_hook(self, hook):
# Clear the cache to make sure the data is up to date:
self._data = None
return self.data().get(hook) == "True"
def trigger_hook(self, hook):
self.data_set(hook, "True")
def clear_hook(self, hook):
self.data_delete(hook)
def type(self):
return self.t.resource_type
@ -550,6 +579,13 @@ class Resource(object):
% six.text_type(self.state))
raise exception.ResourceFailure(exc, self, action)
# This method can be called when we replace a resource, too. In that
# case, a hook has already been dealt with in `Resource.update` so we
# shouldn't do it here again:
if self.stack.action == self.stack.CREATE:
yield self._break_if_required(
self.CREATE, environment.HOOK_PRE_CREATE)
LOG.info(_LI('creating %s'), six.text_type(self))
# Re-resolve the template, since if the resource Ref's
@ -689,6 +725,9 @@ class Resource(object):
after_props = after.properties(self.properties_schema,
self.context)
yield self._break_if_required(
self.UPDATE, environment.HOOK_PRE_UPDATE)
if not self._needs_update(after, before, after_props, before_props,
prev_resource):
return
@ -1081,6 +1120,20 @@ class Resource(object):
return 'alarm state changed to %(state)s' % details
return 'Unknown'
# Clear the hook without interfering with resources'
# `handle_signal` callbacks:
if (details and 'unset_hook' in details and
environment.valid_hook_type(details.get('unset_hook'))):
hook = details['unset_hook']
if self.has_hook(hook):
self.clear_hook(hook)
LOG.info(_LI('Clearing %(hook)s hook on %(resource)s'),
{'hook': hook, 'resource': six.text_type(self)})
self._add_event(self.action, self.status,
"Hook %s is cleared" % hook)
return
if not callable(getattr(self, 'handle_signal', None)):
raise exception.ResourceActionNotSupported(action='signal')

View File

@ -166,6 +166,7 @@ class StackResource(resource.Resource):
child_env = environment.get_child_environment(
self.stack.env, child_params,
child_resource_name=self.name,
item_to_remove=self.resource_info)
parsed_template = self._child_parsed_template(child_template,
@ -229,6 +230,7 @@ class StackResource(resource.Resource):
child_env = environment.get_child_environment(
self.stack.env,
user_params,
child_resource_name=self.name,
item_to_remove=self.resource_info)
new_nested_depth = self._child_nested_depth()
@ -369,7 +371,9 @@ class StackResource(resource.Resource):
child_env = environment.get_child_environment(
self.stack.env,
user_params, item_to_remove=self.resource_info)
user_params,
child_resource_name=self.name,
item_to_remove=self.resource_info)
parsed_template = self._child_parsed_template(child_template,
child_env)

View File

@ -549,3 +549,268 @@ class ChildEnvTest(common.HeatTestCase):
cenv = environment.get_child_environment(penv, None)
res = cenv.get_resource_info('OS::Food', resource_name='abc')
self.assertIsNotNone(res)
def test_drill_down_to_child_resource(self):
env = {
u'resource_registry': {
u'OS::Food': u'fruity.yaml',
u'resources': {
u'a': {
u'OS::Fruit': u'apples.yaml',
u'hooks': 'pre-create',
},
u'nested': {
u'b': {
u'OS::Fruit': u'carrots.yaml',
},
u'nested_res': {
u'hooks': 'pre-create',
}
}
}
}
}
penv = environment.Environment(env)
cenv = environment.get_child_environment(
penv, None, child_resource_name=u'nested')
registry = cenv.user_env_as_dict()['resource_registry']
resources = registry['resources']
self.assertIn('nested_res', resources)
self.assertIn('hooks', resources['nested_res'])
self.assertIsNotNone(
cenv.get_resource_info('OS::Food', resource_name='abc'))
self.assertIsNone(
cenv.get_resource_info('OS::Fruit', resource_name='a'))
res = cenv.get_resource_info('OS::Fruit', resource_name='b')
self.assertIsNotNone(res)
self.assertEqual(u'carrots.yaml', res.value)
def test_drill_down_non_matching_wildcard(self):
env = {
u'resource_registry': {
u'resources': {
u'nested': {
u'c': {
u'OS::Fruit': u'carrots.yaml',
u'hooks': 'pre-create',
},
},
u'*_doesnt_match_nested': {
u'nested_res': {
u'hooks': 'pre-create',
},
}
}
}
}
penv = environment.Environment(env)
cenv = environment.get_child_environment(
penv, None, child_resource_name=u'nested')
registry = cenv.user_env_as_dict()['resource_registry']
resources = registry['resources']
self.assertIn('c', resources)
self.assertNotIn('nested_res', resources)
res = cenv.get_resource_info('OS::Fruit', resource_name='c')
self.assertIsNotNone(res)
self.assertEqual(u'carrots.yaml', res.value)
def test_drill_down_matching_wildcard(self):
env = {
u'resource_registry': {
u'resources': {
u'nested': {
u'c': {
u'OS::Fruit': u'carrots.yaml',
u'hooks': 'pre-create',
},
},
u'nest*': {
u'nested_res': {
u'hooks': 'pre-create',
},
}
}
}
}
penv = environment.Environment(env)
cenv = environment.get_child_environment(
penv, None, child_resource_name=u'nested')
registry = cenv.user_env_as_dict()['resource_registry']
resources = registry['resources']
self.assertIn('c', resources)
self.assertIn('nested_res', resources)
res = cenv.get_resource_info('OS::Fruit', resource_name='c')
self.assertIsNotNone(res)
self.assertEqual(u'carrots.yaml', res.value)
def test_drill_down_prefer_exact_match(self):
env = {
u'resource_registry': {
u'resources': {
u'*esource': {
u'hooks': 'pre-create',
},
u'res*': {
u'hooks': 'pre-create',
},
u'resource': {
u'OS::Fruit': u'carrots.yaml',
u'hooks': 'pre-update',
},
u'resource*': {
u'hooks': 'pre-create',
},
u'*resource': {
u'hooks': 'pre-create',
},
u'*sour*': {
u'hooks': 'pre-create',
},
}
}
}
penv = environment.Environment(env)
cenv = environment.get_child_environment(
penv, None, child_resource_name=u'resource')
registry = cenv.user_env_as_dict()['resource_registry']
resources = registry['resources']
self.assertEqual(u'carrots.yaml', resources[u'OS::Fruit'])
self.assertEqual('pre-update', resources[u'hooks'])
class ResourceRegistryTest(common.HeatTestCase):
def test_resources_load(self):
resources = {
u'pre_create': {
u'OS::Fruit': u'apples.yaml',
u'hooks': 'pre-create',
},
u'pre_update': {
u'hooks': 'pre-update',
},
u'both': {
u'hooks': ['pre-create', 'pre-update'],
},
u'b': {
u'OS::Food': u'fruity.yaml',
},
u'nested': {
u'res': {
u'hooks': 'pre-create',
},
},
}
registry = environment.ResourceRegistry(None, {})
registry.load({'resources': resources})
self.assertIsNotNone(registry.get_resource_info(
'OS::Fruit', resource_name='pre_create'))
self.assertIsNotNone(registry.get_resource_info(
'OS::Food', resource_name='b'))
resources = registry.as_dict()['resources']
self.assertEqual('pre-create',
resources['pre_create']['hooks'])
self.assertEqual('pre-update',
resources['pre_update']['hooks'])
self.assertEqual(['pre-create', 'pre-update'],
resources['both']['hooks'])
self.assertEqual('pre-create',
resources['nested']['res']['hooks'])
class HookMatchTest(common.HeatTestCase):
def test_plain_matches(self):
resources = {
u'a': {
u'OS::Fruit': u'apples.yaml',
u'hooks': [u'pre-create', u'pre-update'],
},
u'b': {
u'OS::Food': u'fruity.yaml',
},
u'nested': {
u'res': {
u'hooks': 'pre-create',
},
},
}
registry = environment.ResourceRegistry(None, {})
registry.load({
u'OS::Fruit': u'apples.yaml',
'resources': resources})
self.assertTrue(registry.matches_hook(
'a', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'b', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'OS::Fruit', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'res', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'unknown', environment.HOOK_PRE_CREATE))
def test_wildcard_matches(self):
resources = {
u'prefix_*': {
u'hooks': 'pre-create',
},
u'*_suffix': {
u'hooks': 'pre-create',
},
u'*': {
u'hooks': 'pre-update',
},
}
registry = environment.ResourceRegistry(None, {})
registry.load({'resources': resources})
self.assertTrue(registry.matches_hook(
'prefix_', environment.HOOK_PRE_CREATE))
self.assertTrue(registry.matches_hook(
'prefix_some', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'some_prefix', environment.HOOK_PRE_CREATE))
self.assertTrue(registry.matches_hook(
'_suffix', environment.HOOK_PRE_CREATE))
self.assertTrue(registry.matches_hook(
'some_suffix', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'_suffix_blah', environment.HOOK_PRE_CREATE))
self.assertTrue(registry.matches_hook(
'some_prefix', environment.HOOK_PRE_UPDATE))
self.assertTrue(registry.matches_hook(
'_suffix_blah', environment.HOOK_PRE_UPDATE))
def test_hook_types(self):
resources = {
u'pre_create': {
u'hooks': 'pre-create',
},
u'pre_update': {
u'hooks': 'pre-update',
},
u'both': {
u'hooks': ['pre-create', 'pre-update'],
},
}
registry = environment.ResourceRegistry(None, {})
registry.load({'resources': resources})
self.assertTrue(registry.matches_hook(
'pre_create', environment.HOOK_PRE_CREATE))
self.assertFalse(registry.matches_hook(
'pre_create', environment.HOOK_PRE_UPDATE))
self.assertTrue(registry.matches_hook(
'pre_update', environment.HOOK_PRE_UPDATE))
self.assertFalse(registry.matches_hook(
'pre_update', environment.HOOK_PRE_CREATE))
self.assertTrue(registry.matches_hook(
'both', environment.HOOK_PRE_CREATE))
self.assertTrue(registry.matches_hook(
'both', environment.HOOK_PRE_UPDATE))

View File

@ -1812,3 +1812,92 @@ class ReducePhysicalResourceNameTest(common.HeatTestCase):
else:
# check that nothing has changed
self.assertEqual(self.original, reduced)
class ResourceHookTest(common.HeatTestCase):
def setUp(self):
super(ResourceHookTest, self).setUp()
resource._register_class('GenericResourceType',
generic_rsrc.GenericResource)
resource._register_class('ResourceWithCustomConstraint',
generic_rsrc.ResourceWithCustomConstraint)
self.env = environment.Environment()
self.env.load({u'resource_registry':
{u'OS::Test::GenericResource': u'GenericResourceType',
u'OS::Test::ResourceWithCustomConstraint':
u'ResourceWithCustomConstraint'}})
self.stack = parser.Stack(utils.dummy_context(), 'test_stack',
parser.Template(empty_template,
env=self.env),
stack_id=str(uuid.uuid4()))
def test_hook(self):
snippet = rsrc_defn.ResourceDefinition('res',
'GenericResourceType')
res = resource.Resource('res', snippet, self.stack)
res.data = mock.Mock(return_value={})
self.assertFalse(res.has_hook('pre-create'))
self.assertFalse(res.has_hook('pre-update'))
res.data = mock.Mock(return_value={'pre-create': 'True'})
self.assertTrue(res.has_hook('pre-create'))
self.assertFalse(res.has_hook('pre-update'))
res.data = mock.Mock(return_value={'pre-create': 'False'})
self.assertFalse(res.has_hook('pre-create'))
self.assertFalse(res.has_hook('pre-update'))
res.data = mock.Mock(return_value={'pre-update': 'True'})
self.assertFalse(res.has_hook('pre-create'))
self.assertTrue(res.has_hook('pre-update'))
def test_set_hook(self):
snippet = rsrc_defn.ResourceDefinition('res',
'GenericResourceType')
res = resource.Resource('res', snippet, self.stack)
res.data_set = mock.Mock()
res.data_delete = mock.Mock()
res.trigger_hook('pre-create')
res.data_set.assert_called_with('pre-create', 'True')
res.trigger_hook('pre-update')
res.data_set.assert_called_with('pre-update', 'True')
res.clear_hook('pre-create')
res.data_delete.assert_called_with('pre-create')
def test_signal_clear_hook(self):
snippet = rsrc_defn.ResourceDefinition('res',
'GenericResourceType')
res = resource.Resource('res', snippet, self.stack)
res.clear_hook = mock.Mock()
res.has_hook = mock.Mock(return_value=True)
self.assertRaises(exception.ResourceActionNotSupported,
res.signal, None)
self.assertFalse(res.clear_hook.called)
self.assertRaises(exception.ResourceActionNotSupported,
res.signal, {})
self.assertFalse(res.clear_hook.called)
self.assertRaises(exception.ResourceActionNotSupported,
res.signal, {'unset_hook': 'unknown_hook'})
self.assertFalse(res.clear_hook.called)
res.signal({'unset_hook': 'pre-create'})
res.clear_hook.assert_called_with('pre-create')
res.signal({'unset_hook': 'pre-update'})
res.clear_hook.assert_called_with('pre-update')
res.has_hook = mock.Mock(return_value=False)
self.assertRaises(exception.ResourceActionNotSupported,
res.signal, {'unset_hook': 'pre-create'})

View File

@ -237,8 +237,11 @@ class StackResourceTest(common.HeatTestCase):
parent_resource._validate_nested_resources = validation_mock
result = parent_resource.preview()
mock_env_class.assert_called_once_with(self.parent_stack.env,
params, item_to_remove=None)
mock_env_class.assert_called_once_with(
self.parent_stack.env,
params,
child_resource_name='test',
item_to_remove=None)
self.assertEqual('preview_nested_stack', result)
mock_stack_class.assert_called_once_with(
mock.ANY,
@ -276,8 +279,11 @@ class StackResourceTest(common.HeatTestCase):
parent_resource._validate_nested_resources = validation_mock
result = parent_resource.preview()
mock_env_class.assert_called_once_with(self.parent_stack.env,
params, item_to_remove=None)
mock_env_class.assert_called_once_with(
self.parent_stack.env,
params,
child_resource_name='test',
item_to_remove=None)
self.assertEqual('preview_nested_stack', result)
mock_stack_class.assert_called_once_with(
mock.ANY,