From 2f563535a21a82b6e18f6116bd0a4e4d31119904 Mon Sep 17 00:00:00 2001 From: Bill Arnold Date: Fri, 22 Aug 2014 13:55:14 -0400 Subject: [PATCH] Implements stack lifecycle plugpoints Stack lifecycle plugpoints have been proposed in http://summit.openstack.org/cfp/details/86 and https://etherpad.openstack.org/p/juno-summit-heat-callbacks Implements: blueprint stack-lifecycle-plugpoint Change-Id: I8c7b5d0113392e54fe0f35933c2c10da277fd90b --- heat/common/lifecycle_plugin_utils.py | 117 ++++++++++ heat/engine/environment.py | 9 + heat/engine/lifecycle_plugin.py | 54 +++++ heat/engine/resources/__init__.py | 9 + heat/engine/stack.py | 37 ++++ heat/tests/test_lifecycle_plugin_utils.py | 258 ++++++++++++++++++++++ setup.cfg | 2 + 7 files changed, 486 insertions(+) create mode 100755 heat/common/lifecycle_plugin_utils.py create mode 100755 heat/engine/lifecycle_plugin.py create mode 100755 heat/tests/test_lifecycle_plugin_utils.py diff --git a/heat/common/lifecycle_plugin_utils.py b/heat/common/lifecycle_plugin_utils.py new file mode 100755 index 000000000..78f4e47c8 --- /dev/null +++ b/heat/common/lifecycle_plugin_utils.py @@ -0,0 +1,117 @@ + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +Utility for fetching and running plug point implementation classes +''' +from heat.engine import resources +from heat.openstack.common.gettextutils import _LE +from heat.openstack.common import log as logging + +LOG = logging.getLogger(__name__) +pp_class_instances = None + + +def get_plug_point_class_instances(): + ''' + Get list of instances of classes that (may) implement pre and post + stack operation methods. + + The list of class instances is sorted using get_ordinal methods + on the plug point classes. If class1.ordinal() < class2.ordinal(), + then class1 will be before before class2 in the list. + ''' + global pp_class_instances + if pp_class_instances is None: + pp_class_instances = [] + pp_classes = [] + try: + slps = resources.global_env().get_stack_lifecycle_plugins() + pp_classes = [cls for name, cls in slps] + except Exception: + LOG.exception(_LE("failed to get lifecycle plug point classes")) + + for ppc in pp_classes: + try: + pp_class_instances.append(ppc()) + except Exception: + LOG.exception( + _LE("failed to instantiate stack lifecycle class %s"), ppc) + try: + pp_class_instances = sorted(pp_class_instances, + key=lambda ppci: ppci.get_ordinal()) + except Exception: + LOG.exception(_LE("failed to sort lifecycle plug point classes")) + return pp_class_instances + + +def do_pre_ops(cnxt, stack, current_stack=None, action=None): + ''' + Call available pre-op methods sequentially, in order determined with + get_ordinal(), with parameters context, stack, current_stack, action + + On failure of any pre_op method, will call post-op methods corresponding + to successful calls of pre-op methods + ''' + cinstances = get_plug_point_class_instances() + if action is None: + action = stack.action + failure, failure_exception_message, success_count = _do_ops( + cinstances, 'do_pre_op', cnxt, stack, current_stack, action, None) + + if failure: + cinstances = cinstances[0:success_count] + _do_ops(cinstances, 'do_post_op', cnxt, stack, current_stack, + action, True) + raise Exception(failure_exception_message) + + +def do_post_ops(cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + ''' + Call available post-op methods sequentially, in order determined with + get_ordinal(), with parameters context, stack, current_stack, + action, is_stack_failure + ''' + cinstances = get_plug_point_class_instances() + if action is None: + action = stack.action + _do_ops(cinstances, 'do_post_op', cnxt, stack, current_stack, action, None) + + +def _do_ops(cinstances, opname, cnxt, stack, current_stack=None, action=None, + is_stack_failure=None): + success_count = 0 + failure = False + failure_exception_message = None + for ci in cinstances: + op = getattr(ci, opname, None) + if callable(op): + try: + if is_stack_failure is not None: + op(cnxt, stack, current_stack, action, is_stack_failure) + else: + op(cnxt, stack, current_stack, action) + success_count += 1 + except Exception as ex: + LOG.exception(_LE( + "%(opname) %(ci)s failed for %(a)s on %(sid)s") % + {'opname': opname, 'ci': type(ci), + 'a': action, 'sid': stack.id}) + failure = True + failure_exception_message = ex.args[0] if ex.args else str(ex) + break + LOG.info(_("done with class=%(c)s, stackid=%(sid)s, action=%(a)s") % + {'c': type(ci), 'sid': stack.id, 'a': action}) + return (failure, failure_exception_message, success_count) diff --git a/heat/engine/environment.py b/heat/engine/environment.py index b40b70265..43e22c30f 100644 --- a/heat/engine/environment.py +++ b/heat/engine/environment.py @@ -364,6 +364,7 @@ class Environment(object): self.params = dict((k, v) for (k, v) in six.iteritems(env) if k != RESOURCE_REGISTRY) self.constraints = {} + self.stack_lifecycle_plugins = [] def load(self, env_snippet): self.registry.load(env_snippet.get(RESOURCE_REGISTRY, {})) @@ -380,6 +381,11 @@ class Environment(object): def register_constraint(self, constraint_name, constraint): self.constraints[constraint_name] = constraint + def register_stack_lifecycle_plugin(self, stack_lifecycle_name, + stack_lifecycle_class): + self.stack_lifecycle_plugins.append((stack_lifecycle_name, + stack_lifecycle_class)) + def get_class(self, resource_type, resource_name=None): return self.registry.get_class(resource_type, resource_name) @@ -394,6 +400,9 @@ class Environment(object): def get_constraint(self, name): return self.constraints.get(name) + def get_stack_lifecycle_plugins(self): + return self.stack_lifecycle_plugins + def read_global_environment(env, env_dir=None): if env_dir is None: diff --git a/heat/engine/lifecycle_plugin.py b/heat/engine/lifecycle_plugin.py new file mode 100755 index 000000000..482660d6a --- /dev/null +++ b/heat/engine/lifecycle_plugin.py @@ -0,0 +1,54 @@ + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class LifecyclePlugin(object): + ''' + base class for pre-op and post-op work on a stack + Implementations should extend this class and override the methods + ''' + def do_pre_op(self, cnxt, stack, current_stack=None, action=None): + ''' + method to be run by heat before stack operations + ''' + pass + + def do_post_op(self, cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + ''' + Method to be run by heat after stack operations, including failures. + + On failure to execute all the registered pre_ops, this method will be + called if and only if the corresponding pre_op was successfully called. + On failures of the actual stack operation, this method will + be called if all the pre operations were successfully called. + ''' + pass + + def get_ordinal(self): + ''' + An ordinal used to order class instances for pre and post + operation execution. + + The values returned by get_ordinal are used to create a partial order + for pre and post operation method invocations. The default ordinal + value of 100 may be overridden. + If class1inst.ordinal() < class2inst.ordinal(), then the method on + class1inst will be executed before the method on class2inst. + If class1inst.ordinal() > class2inst.ordinal(), then the method on + class1inst will be executed after the method on class2inst. + If class1inst.ordinal() == class2inst.ordinal(), then the order of + method invocation is indeterminate. + ''' + return 100 diff --git a/heat/engine/resources/__init__.py b/heat/engine/resources/__init__.py index 5a9ad8977..5da6024b4 100644 --- a/heat/engine/resources/__init__.py +++ b/heat/engine/resources/__init__.py @@ -28,6 +28,12 @@ def _register_constraints(env, type_pairs): env.register_constraint(constraint_name, constraint) +def _register_stack_lifecycle_plugins(env, type_pairs): + for stack_lifecycle_name, stack_lifecycle_class in type_pairs: + env.register_stack_lifecycle_plugin(stack_lifecycle_name, + stack_lifecycle_class) + + def _get_mapping(namespace): mgr = extension.ExtensionManager( namespace=namespace, @@ -64,6 +70,9 @@ def _load_global_environment(env): def _load_global_resources(env): _register_constraints(env, _get_mapping('heat.constraints')) + _register_stack_lifecycle_plugins( + env, + _get_mapping('heat.stack_lifecycle_plugins')) manager = plugin_manager.PluginManager(__name__) # Sometimes resources should not be available for registration in Heat due diff --git a/heat/engine/stack.py b/heat/engine/stack.py index edc6bbbc3..993c5a835 100644 --- a/heat/engine/stack.py +++ b/heat/engine/stack.py @@ -24,6 +24,7 @@ from heat.common import context as common_context from heat.common import exception from heat.common.exception import StackValidationFailed from heat.common import identifier +from heat.common import lifecycle_plugin_utils from heat.db import api as db_api from heat.engine import dependencies from heat.engine import environment @@ -545,6 +546,15 @@ class Stack(collections.Mapping): A task to perform an action on the stack and all of the resources in forward or reverse dependency order as specified by reverse ''' + try: + lifecycle_plugin_utils.do_pre_ops(self.context, self, + None, action) + except Exception as e: + self.state_set(action, self.FAILED, e.args[0] if e.args else + 'Failed stack pre-ops: %s' % six.text_type(e)) + if callable(post_func): + post_func() + return self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % action) @@ -581,6 +591,8 @@ class Stack(collections.Mapping): if callable(post_func): post_func() + lifecycle_plugin_utils.do_post_ops(self.context, self, None, action, + (self.status == self.FAILED)) def check(self): self.updated_time = datetime.utcnow() @@ -667,6 +679,13 @@ class Stack(collections.Mapping): "Invalid action %s" % action) return + try: + lifecycle_plugin_utils.do_pre_ops(self.context, self, + newstack, action) + except Exception as e: + self.state_set(action, self.FAILED, e.args[0] if e.args else + 'Failed stack pre-ops: %s' % six.text_type(e)) + return if self.status == self.IN_PROGRESS: if action == self.ROLLBACK: LOG.debug("Starting update rollback for %s" % self.name) @@ -739,6 +758,9 @@ class Stack(collections.Mapping): self.status_reason = reason self.store() + lifecycle_plugin_utils.do_post_ops(self.context, self, + newstack, action, + (self.status == self.FAILED)) notification.send(self) @@ -756,6 +778,8 @@ class Stack(collections.Mapping): "Invalid action %s" % action) return + # Note abandon is a delete with + # stack.set_deletion_policy(resource.RETAIN) stack_status = self.COMPLETE reason = 'Stack %s completed successfully' % action self.state_set(action, self.IN_PROGRESS, 'Stack %s started' % @@ -810,6 +834,15 @@ class Stack(collections.Mapping): for snapshot in snapshots: self.delete_snapshot(snapshot) + if not backup: + try: + lifecycle_plugin_utils.do_pre_ops(self.context, self, + None, action) + except Exception as e: + self.state_set(action, self.FAILED, + e.args[0] if e.args else + 'Failed stack pre-ops: %s' % six.text_type(e)) + return action_task = scheduler.DependencyTaskGroup(self.dependencies, resource.Resource.destroy, reverse=True) @@ -874,6 +907,10 @@ class Stack(collections.Mapping): LOG.info(_("Tried to delete stack that does not exist " "%s ") % self.id) + if not backup: + lifecycle_plugin_utils.do_post_ops(self.context, self, + None, action, + (self.status == self.FAILED)) if stack_status != self.FAILED: # delete the stack try: diff --git a/heat/tests/test_lifecycle_plugin_utils.py b/heat/tests/test_lifecycle_plugin_utils.py new file mode 100755 index 000000000..2b60cb94c --- /dev/null +++ b/heat/tests/test_lifecycle_plugin_utils.py @@ -0,0 +1,258 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from heat.common import lifecycle_plugin_utils +from heat.engine import lifecycle_plugin +from heat.engine import resources +from heat.tests.common import HeatTestCase + + +empty_template = ''' +heat_template_version: '2013-05-23' +description: Empty stack +resources: +''' + + +class LifecyclePluginUtilsTests(HeatTestCase): + """ + Basic tests for the helper methods in + :module:'heat.common.lifecycle_plugin_utils'. + """ + + def setUp(self): + super(LifecyclePluginUtilsTests, self).setUp() + + def tearDown(self): + super(LifecyclePluginUtilsTests, self).tearDown() + lifecycle_plugin_utils.pp_class_instances = None + + def mock_lcp_class_map(self, lcp_mappings): + self.m.UnsetStubs() + self.m.StubOutWithMock(resources.global_env(), + 'get_stack_lifecycle_plugins') + resources.global_env().get_stack_lifecycle_plugins().\ + MultipleTimes().AndReturn(lcp_mappings) + self.m.ReplayAll() + # reset cache + lifecycle_plugin_utils.pp_class_instances = None + + def test_get_plug_point_class_instances(self): + """Tests the get_plug_point_class_instances function.""" + lcp_mappings = [('A::B::C1', TestLifecycleCallout1)] + self.mock_lcp_class_map(lcp_mappings) + + pp_cinstances = lifecycle_plugin_utils.get_plug_point_class_instances() + self.assertIsNotNone(pp_cinstances) + self.assertTrue(self.is_iterable(pp_cinstances), + "not iterable: %s" % pp_cinstances) + self.assertTrue(len(pp_cinstances) == 1) + self.assertEqual(TestLifecycleCallout1, pp_cinstances[0].__class__) + + def test_do_pre_and_post_callouts(self): + lcp_mappings = [('A::B::C1', TestLifecycleCallout1)] + self.mock_lcp_class_map(lcp_mappings) + mc = mock.Mock() + mc.__setattr__("pre_counter_for_unit_test", 0) + mc.__setattr__("post_counter_for_unit_test", 0) + ms = mock.Mock() + ms.__setattr__("action", 'A') + lifecycle_plugin_utils.do_pre_ops(mc, ms, None, None) + self.assertEqual(mc.pre_counter_for_unit_test, 1) + lifecycle_plugin_utils.do_post_ops(mc, ms, None, None) + self.assertEqual(mc.post_counter_for_unit_test, 1) + + return + + def test_class_instantiation_and_sorting(self): + lcp_mappings = [] + self.mock_lcp_class_map(lcp_mappings) + pp_cis = lifecycle_plugin_utils.get_plug_point_class_instances() + self.assertEqual(len(pp_cis), 0) + + # order should change with sort + lcp_mappings = [('A::B::C2', TestLifecycleCallout2), + ('A::B::C1', TestLifecycleCallout1)] + self.mock_lcp_class_map(lcp_mappings) + pp_cis = lifecycle_plugin_utils.get_plug_point_class_instances() + self.assertEqual(len(pp_cis), 2) + self.assertEqual(pp_cis[0].get_ordinal(), 100) + self.assertEqual(pp_cis[1].get_ordinal(), 101) + self.assertEqual(TestLifecycleCallout1, pp_cis[0].__class__) + self.assertEqual(TestLifecycleCallout2, pp_cis[1].__class__) + + # order should NOT change with sort + lcp_mappings = [('A::B::C1', TestLifecycleCallout1), + ('A::B::C2', TestLifecycleCallout2)] + self.mock_lcp_class_map(lcp_mappings) + pp_cis = lifecycle_plugin_utils.get_plug_point_class_instances() + self.assertEqual(len(pp_cis), 2) + self.assertEqual(pp_cis[0].get_ordinal(), 100) + self.assertEqual(pp_cis[1].get_ordinal(), 101) + self.assertEqual(TestLifecycleCallout1, pp_cis[0].__class__) + self.assertEqual(TestLifecycleCallout2, pp_cis[1].__class__) + + # sort failure due to exception in thrown by ordinal + lcp_mappings = [('A::B::C2', TestLifecycleCallout2), + ('A::B::C3', TestLifecycleCallout3), + ('A::B::C1', TestLifecycleCallout1)] + self.mock_lcp_class_map(lcp_mappings) + pp_cis = lifecycle_plugin_utils.get_plug_point_class_instances() + self.assertEqual(len(pp_cis), 3) + self.assertEqual(pp_cis[2].get_ordinal(), 100) + self.assertEqual(pp_cis[0].get_ordinal(), 101) + # (can sort fail partially? If so then this test may break) + self.assertEqual(TestLifecycleCallout2, pp_cis[0].__class__) + self.assertEqual(TestLifecycleCallout3, pp_cis[1].__class__) + self.assertEqual(TestLifecycleCallout1, pp_cis[2].__class__) + + return + + def test_do_pre_op_failure(self): + lcp_mappings = [('A::B::C5', TestLifecycleCallout1), + ('A::B::C4', TestLifecycleCallout4)] + self.mock_lcp_class_map(lcp_mappings) + mc = mock.Mock() + mc.__setattr__("pre_counter_for_unit_test", 0) + mc.__setattr__("post_counter_for_unit_test", 0) + ms = mock.Mock() + ms.__setattr__("action", 'A') + failed = False + try: + lifecycle_plugin_utils.do_pre_ops(mc, ms, None, None) + except Exception: + failed = True + self.assertTrue(failed) + self.assertEqual(mc.pre_counter_for_unit_test, 1) + self.assertEqual(mc.post_counter_for_unit_test, 1) + + return + + def test_do_post_op_failure(self): + lcp_mappings = [('A::B::C1', TestLifecycleCallout1), + ('A::B::C5', TestLifecycleCallout5)] + self.mock_lcp_class_map(lcp_mappings) + mc = mock.Mock() + mc.__setattr__("pre_counter_for_unit_test", 0) + mc.__setattr__("post_counter_for_unit_test", 0) + ms = mock.Mock() + ms.__setattr__("action", 'A') + lifecycle_plugin_utils.do_post_ops(mc, ms, None, None) + self.assertEqual(mc.post_counter_for_unit_test, 1) + + return + + def test_exercise_base_lifecycle_plugin_class(self): + lcp = lifecycle_plugin.LifecyclePlugin() + ordinal = lcp.get_ordinal() + lcp.do_pre_op(None, None, None) + lcp.do_post_op(None, None, None) + self.assertEqual(ordinal, 100) + + return + + def is_iterable(self, obj): + #special case string + if not object: + return False + if isinstance(obj, str): + return False + + #Test for iterabilityy + try: + for m in obj: + break + except TypeError: + return False + return True + + +class TestLifecycleCallout1(lifecycle_plugin.LifecyclePlugin): + ''' + Sample test class for testing pre-op and post-op work on a stack + ''' + def do_pre_op(self, cnxt, stack, current_stack=None, action=None): + cnxt.pre_counter_for_unit_test += 1 + + def do_post_op(self, cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + cnxt.post_counter_for_unit_test += 1 + + def get_ordinal(self): + return 100 + + +class TestLifecycleCallout2(lifecycle_plugin.LifecyclePlugin): + ''' + Sample test class for testing pre-op and post-op work on a stack, + different ordinal, and increment counters by 2 + ''' + def do_pre_op(self, cnxt, stack, current_stack=None, action=None): + cnxt.pre_counter_for_unit_test += 2 + + def do_post_op(self, cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + cnxt.post_counter_for_unit_test += 2 + + def get_ordinal(self): + return 101 + + +class TestLifecycleCallout3(lifecycle_plugin.LifecyclePlugin): + ''' + Sample test class for testing pre-op and post-op work on a stack, + methods raise exceptions + ''' + def do_pre_op(self, cnxt, stack, current_stack=None, action=None): + raise Exception() + + def do_post_op(self, cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + raise Exception() + + def get_ordinal(self): + raise Exception() + + +class TestLifecycleCallout4(lifecycle_plugin.LifecyclePlugin): + ''' + Sample test class for testing pre-op and post-op work on a stack; + do_pre_op, do_post_op both throw exception + ''' + def do_pre_op(self, cnxt, stack, current_stack=None, action=None): + raise Exception() + + def do_post_op(self, cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + raise Exception() + + def get_ordinal(self): + return 103 + + +class TestLifecycleCallout5(lifecycle_plugin.LifecyclePlugin): + ''' + Sample test class for testing pre-op and post-op work on a stack; + do_post_op throws exception + ''' + def do_pre_op(self, cnxt, stack, current_stack=None, action=None): + cnxt.pre_counter_for_unit_test += 1 + + def do_post_op(self, cnxt, stack, current_stack=None, action=None, + is_stack_failure=False): + raise Exception() + + def get_ordinal(self): + return 100 diff --git a/setup.cfg b/setup.cfg index 884737aa9..1db0ec2fd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,6 +59,8 @@ heat.constraints = iso_8601 = heat.engine.resources.iso_8601:ISO8601Constraint nova.keypair = heat.engine.resources.nova_keypair:KeypairConstraint +heat.stack_lifecycle_plugins = + heat.templates = heat_template_version.2013-05-23 = heat.engine.hot.template:HOTemplate heat_template_version.2014-10-16 = heat.engine.hot.template:HOTemplate