diff --git a/heat/engine/check_resource.py b/heat/engine/check_resource.py index a2f6d842c2..7d12ceeec7 100644 --- a/heat/engine/check_resource.py +++ b/heat/engine/check_resource.py @@ -163,6 +163,8 @@ class CheckResource(object): return True except exception.UpdateInProgress: + LOG.debug('Waiting for existing update to unlock resource %s', + rsrc.id) if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id): rpc_data = sync_point.serialize_input_data(self.input_data) self._rpc_client.check_resource(cnxt, @@ -170,6 +172,8 @@ class CheckResource(object): current_traversal, rpc_data, is_update, adopt_stack_data) + else: + rsrc.handle_preempt() except exception.ResourceFailure as ex: action = ex.action or rsrc.action reason = 'Resource %s failed: %s' % (action, diff --git a/heat/engine/resource.py b/heat/engine/resource.py index c383f0abf5..0d08d9569a 100644 --- a/heat/engine/resource.py +++ b/heat/engine/resource.py @@ -1456,6 +1456,23 @@ class Resource(status.ResourceStatus): new_requires=new_requires) runner(timeout=timeout, progress_callback=progress_callback) + def handle_preempt(self): + """Pre-empt an in-progress update when a new update is available. + + This method is called when a previous convergence update is in + progress but a new update for the resource is available. By default + it does nothing, but subclasses may override it to cancel the + in-progress update if it is safe to do so. + + Note that this method does not run in the context of the in-progress + update and has no access to runtime information about it; nor is it + safe to make changes to the Resource in the database. If implemented, + this method should cause the existing update to complete by external + means. If this leaves the resource in a FAILED state, that should be + taken into account in needs_replace_failed(). + """ + return + def preview_update(self, after, before, after_props, before_props, prev_resource, check_init_complete=False): """Simulates update without actually updating the resource. diff --git a/heat/engine/resources/stack_resource.py b/heat/engine/resources/stack_resource.py index 7def946c51..bb020c06b3 100644 --- a/heat/engine/resources/stack_resource.py +++ b/heat/engine/resources/stack_resource.py @@ -561,9 +561,10 @@ class StackResource(resource.Resource): return self._check_status_complete(target_action, cookie=cookie) - def handle_update_cancel(self, cookie): + def _handle_cancel(self): stack_identity = self.nested_identifier() if stack_identity is not None: + LOG.debug('Cancelling %s of %s' % (self.action, self)) try: self.rpc_client().stack_cancel_update( self.context, @@ -573,6 +574,12 @@ class StackResource(resource.Resource): LOG.debug('Nested stack %s not in cancellable state', stack_identity.stack_name) + def handle_preempt(self): + self._handle_cancel() + + def handle_update_cancel(self, cookie): + self._handle_cancel() + def handle_create_cancel(self, cookie): return self.handle_update_cancel(cookie) diff --git a/heat_integrationtests/functional/test_simultaneous_update.py b/heat_integrationtests/functional/test_simultaneous_update.py index 0c562c0758..004145d3bf 100644 --- a/heat_integrationtests/functional/test_simultaneous_update.py +++ b/heat_integrationtests/functional/test_simultaneous_update.py @@ -12,6 +12,7 @@ import copy +import json import time from heat_integrationtests.common import test @@ -91,3 +92,85 @@ class SimultaneousUpdateStackTest(functional_base.FunctionalTestsBase): time.sleep(50) self.update_stack(stack_id, after) + + +input_param = 'input' +preempt_nested_stack_type = 'preempt.yaml' +preempt_root_rsrcs = { + 'nested_stack': { + 'type': preempt_nested_stack_type, + 'properties': { + 'input': {'get_param': input_param}, + }, + } +} +preempt_root_out = {'get_attr': ['nested_stack', 'delay_stack']} +preempt_delay_stack_type = 'delay.yaml' +preempt_nested_rsrcs = { + 'delay_stack': { + 'type': preempt_delay_stack_type, + 'properties': { + 'input': {'get_param': input_param}, + }, + } +} +preempt_nested_out = {'get_resource': 'delay_stack'} +preempt_delay_rsrcs = { + 'delay_resource': { + 'type': 'OS::Heat::TestResource', + 'properties': { + 'action_wait_secs': { + 'update': 6000, + }, + 'value': {'get_param': input_param}, + }, + } +} + + +def _tmpl_with_rsrcs(rsrcs, output_value=None): + tmpl = { + 'heat_template_version': 'queens', + 'parameters': { + input_param: { + 'type': 'string', + }, + }, + 'resources': rsrcs, + } + if output_value is not None: + outputs = {'delay_stack': {'value': output_value}} + tmpl['outputs'] = outputs + return json.dumps(tmpl) + + +class SimultaneousUpdateNestedStackTest(functional_base.FunctionalTestsBase): + @test.requires_convergence + def test_nested_preemption(self): + root_tmpl = _tmpl_with_rsrcs(preempt_root_rsrcs, + preempt_root_out) + files = { + preempt_nested_stack_type: _tmpl_with_rsrcs(preempt_nested_rsrcs, + preempt_nested_out), + preempt_delay_stack_type: _tmpl_with_rsrcs(preempt_delay_rsrcs), + } + stack_id = self.stack_create(template=root_tmpl, files=files, + parameters={input_param: 'foo'}) + delay_stack_uuid = self.get_stack_output(stack_id, 'delay_stack') + + # Start an update that includes a long delay in the second nested stack + self.update_stack(stack_id, template=root_tmpl, files=files, + parameters={input_param: 'bar'}, + expected_status='UPDATE_IN_PROGRESS') + self._wait_for_resource_status(delay_stack_uuid, 'delay_resource', + 'UPDATE_IN_PROGRESS') + + # Update again to check that we preempt update of the first nested + # stack. This will delete the second nested stack, after preempting the + # update of that stack as well, which will cause the delay resource + # within to be cancelled. + empty_nest_files = { + preempt_nested_stack_type: _tmpl_with_rsrcs({}), + } + self.update_stack(stack_id, template=root_tmpl, files=empty_nest_files, + parameters={input_param: 'baz'})