Pre-empt in-progress nested stack updates on new update

If the parent resource of a nested stack is locked due to an IN_PROGRESS
update, cancel the nested stack update (which will result in the parent
resource being marked FAILED and releasing the lock so that the new
traversal can begin acting on it). This also cancels all descendants of
the nested stack.

This means that a concurrent update no longer gets blocked at a nested
stack boundary until the previous update has finished.

Change-Id: I5f14453ebab75d89672c6eea12de46d48a5147f3
Task: 17760

heat/engine/check_resource.py

@@ -163,6 +163,8 @@ class CheckResource(object):
 
             return True
         except exception.UpdateInProgress:
+            LOG.debug('Waiting for existing update to unlock resource %s',
+                      rsrc.id)
             if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id):
                 rpc_data = sync_point.serialize_input_data(self.input_data)
                 self._rpc_client.check_resource(cnxt,
@@ -170,6 +172,8 @@ class CheckResource(object):
                                                 current_traversal,
                                                 rpc_data, is_update,
                                                 adopt_stack_data)
+            else:
+                rsrc.handle_preempt()
         except exception.ResourceFailure as ex:
             action = ex.action or rsrc.action
             reason = 'Resource %s failed: %s' % (action,

heat/engine/resource.py

@@ -1456,6 +1456,23 @@ class Resource(status.ResourceStatus):
                                       new_requires=new_requires)
         runner(timeout=timeout, progress_callback=progress_callback)
 
+    def handle_preempt(self):
+        """Pre-empt an in-progress update when a new update is available.
+
+        This method is called when a previous convergence update is in
+        progress but a new update for the resource is available. By default
+        it does nothing, but subclasses may override it to cancel the
+        in-progress update if it is safe to do so.
+
+        Note that this method does not run in the context of the in-progress
+        update and has no access to runtime information about it; nor is it
+        safe to make changes to the Resource in the database. If implemented,
+        this method should cause the existing update to complete by external
+        means. If this leaves the resource in a FAILED state, that should be
+        taken into account in needs_replace_failed().
+        """
+        return
+
     def preview_update(self, after, before, after_props, before_props,
                        prev_resource, check_init_complete=False):
         """Simulates update without actually updating the resource.

heat/engine/resources/stack_resource.py

@@ -561,9 +561,10 @@ class StackResource(resource.Resource):
         return self._check_status_complete(target_action,
                                            cookie=cookie)
 
-    def handle_update_cancel(self, cookie):
+    def _handle_cancel(self):
         stack_identity = self.nested_identifier()
         if stack_identity is not None:
+            LOG.debug('Cancelling %s of %s' % (self.action, self))
             try:
                 self.rpc_client().stack_cancel_update(
                     self.context,
@@ -573,6 +574,12 @@ class StackResource(resource.Resource):
                 LOG.debug('Nested stack %s not in cancellable state',
                           stack_identity.stack_name)
 
+    def handle_preempt(self):
+        self._handle_cancel()
+
+    def handle_update_cancel(self, cookie):
+        self._handle_cancel()
+
     def handle_create_cancel(self, cookie):
         return self.handle_update_cancel(cookie)
 

heat_integrationtests/functional/test_simultaneous_update.py

@@ -12,6 +12,7 @@
 
 
 import copy
+import json
 import time
 
 from heat_integrationtests.common import test
@@ -91,3 +92,85 @@ class SimultaneousUpdateStackTest(functional_base.FunctionalTestsBase):
         time.sleep(50)
 
         self.update_stack(stack_id, after)
+
+
+input_param = 'input'
+preempt_nested_stack_type = 'preempt.yaml'
+preempt_root_rsrcs = {
+    'nested_stack': {
+        'type': preempt_nested_stack_type,
+        'properties': {
+            'input': {'get_param': input_param},
+        },
+    }
+}
+preempt_root_out = {'get_attr': ['nested_stack', 'delay_stack']}
+preempt_delay_stack_type = 'delay.yaml'
+preempt_nested_rsrcs = {
+    'delay_stack': {
+        'type': preempt_delay_stack_type,
+        'properties': {
+            'input': {'get_param': input_param},
+        },
+    }
+}
+preempt_nested_out = {'get_resource': 'delay_stack'}
+preempt_delay_rsrcs = {
+    'delay_resource': {
+        'type': 'OS::Heat::TestResource',
+        'properties': {
+            'action_wait_secs': {
+                'update': 6000,
+            },
+            'value': {'get_param': input_param},
+        },
+    }
+}
+
+
+def _tmpl_with_rsrcs(rsrcs, output_value=None):
+    tmpl = {
+        'heat_template_version': 'queens',
+        'parameters': {
+            input_param: {
+                'type': 'string',
+            },
+        },
+        'resources': rsrcs,
+    }
+    if output_value is not None:
+        outputs = {'delay_stack': {'value': output_value}}
+        tmpl['outputs'] = outputs
+    return json.dumps(tmpl)
+
+
+class SimultaneousUpdateNestedStackTest(functional_base.FunctionalTestsBase):
+    @test.requires_convergence
+    def test_nested_preemption(self):
+        root_tmpl = _tmpl_with_rsrcs(preempt_root_rsrcs,
+                                     preempt_root_out)
+        files = {
+            preempt_nested_stack_type: _tmpl_with_rsrcs(preempt_nested_rsrcs,
+                                                        preempt_nested_out),
+            preempt_delay_stack_type: _tmpl_with_rsrcs(preempt_delay_rsrcs),
+        }
+        stack_id = self.stack_create(template=root_tmpl, files=files,
+                                     parameters={input_param: 'foo'})
+        delay_stack_uuid = self.get_stack_output(stack_id, 'delay_stack')
+
+        # Start an update that includes a long delay in the second nested stack
+        self.update_stack(stack_id, template=root_tmpl, files=files,
+                          parameters={input_param: 'bar'},
+                          expected_status='UPDATE_IN_PROGRESS')
+        self._wait_for_resource_status(delay_stack_uuid, 'delay_resource',
+                                       'UPDATE_IN_PROGRESS')
+
+        # Update again to check that we preempt update of the first nested
+        # stack. This will delete the second nested stack, after preempting the
+        # update of that stack as well, which will cause the delay resource
+        # within to be cancelled.
+        empty_nest_files = {
+            preempt_nested_stack_type: _tmpl_with_rsrcs({}),
+        }
+        self.update_stack(stack_id, template=root_tmpl, files=empty_nest_files,
+                          parameters={input_param: 'baz'})