Merge "Pre-empt in-progress nested stack updates on new update"

heat/engine/check_resource.py

@@ -163,6 +163,8 @@ class CheckResource(object):
 
             return True
         except exception.UpdateInProgress:
+            LOG.debug('Waiting for existing update to unlock resource %s',
+                      rsrc.id)
             if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id):
                 rpc_data = sync_point.serialize_input_data(self.input_data)
                 self._rpc_client.check_resource(cnxt,
@@ -170,6 +172,8 @@ class CheckResource(object):
                                                 current_traversal,
                                                 rpc_data, is_update,
                                                 adopt_stack_data)
+            else:
+                rsrc.handle_preempt()
         except exception.ResourceFailure as ex:
             action = ex.action or rsrc.action
             reason = 'Resource %s failed: %s' % (action,

heat/engine/resource.py

@@ -1456,6 +1456,23 @@ class Resource(status.ResourceStatus):
                                       new_requires=new_requires)
         runner(timeout=timeout, progress_callback=progress_callback)
 
+    def handle_preempt(self):
+        """Pre-empt an in-progress update when a new update is available.
+
+        This method is called when a previous convergence update is in
+        progress but a new update for the resource is available. By default
+        it does nothing, but subclasses may override it to cancel the
+        in-progress update if it is safe to do so.
+
+        Note that this method does not run in the context of the in-progress
+        update and has no access to runtime information about it; nor is it
+        safe to make changes to the Resource in the database. If implemented,
+        this method should cause the existing update to complete by external
+        means. If this leaves the resource in a FAILED state, that should be
+        taken into account in needs_replace_failed().
+        """
+        return
+
     def preview_update(self, after, before, after_props, before_props,
                        prev_resource, check_init_complete=False):
         """Simulates update without actually updating the resource.

heat/engine/resources/stack_resource.py

@@ -561,9 +561,10 @@ class StackResource(resource.Resource):
         return self._check_status_complete(target_action,
                                            cookie=cookie)
 
-    def handle_update_cancel(self, cookie):
+    def _handle_cancel(self):
         stack_identity = self.nested_identifier()
         if stack_identity is not None:
+            LOG.debug('Cancelling %s of %s' % (self.action, self))
             try:
                 self.rpc_client().stack_cancel_update(
                     self.context,
@@ -573,6 +574,12 @@ class StackResource(resource.Resource):
                 LOG.debug('Nested stack %s not in cancellable state',
                           stack_identity.stack_name)
 
+    def handle_preempt(self):
+        self._handle_cancel()
+
+    def handle_update_cancel(self, cookie):
+        self._handle_cancel()
+
     def handle_create_cancel(self, cookie):
         return self.handle_update_cancel(cookie)
 

heat_integrationtests/functional/test_simultaneous_update.py

@@ -12,6 +12,7 @@
 
 
 import copy
+import json
 import time
 
 from heat_integrationtests.common import test
@@ -91,3 +92,85 @@ class SimultaneousUpdateStackTest(functional_base.FunctionalTestsBase):
         time.sleep(50)
 
         self.update_stack(stack_id, after)
+
+
+input_param = 'input'
+preempt_nested_stack_type = 'preempt.yaml'
+preempt_root_rsrcs = {
+    'nested_stack': {
+        'type': preempt_nested_stack_type,
+        'properties': {
+            'input': {'get_param': input_param},
+        },
+    }
+}
+preempt_root_out = {'get_attr': ['nested_stack', 'delay_stack']}
+preempt_delay_stack_type = 'delay.yaml'
+preempt_nested_rsrcs = {
+    'delay_stack': {
+        'type': preempt_delay_stack_type,
+        'properties': {
+            'input': {'get_param': input_param},
+        },
+    }
+}
+preempt_nested_out = {'get_resource': 'delay_stack'}
+preempt_delay_rsrcs = {
+    'delay_resource': {
+        'type': 'OS::Heat::TestResource',
+        'properties': {
+            'action_wait_secs': {
+                'update': 6000,
+            },
+            'value': {'get_param': input_param},
+        },
+    }
+}
+
+
+def _tmpl_with_rsrcs(rsrcs, output_value=None):
+    tmpl = {
+        'heat_template_version': 'queens',
+        'parameters': {
+            input_param: {
+                'type': 'string',
+            },
+        },
+        'resources': rsrcs,
+    }
+    if output_value is not None:
+        outputs = {'delay_stack': {'value': output_value}}
+        tmpl['outputs'] = outputs
+    return json.dumps(tmpl)
+
+
+class SimultaneousUpdateNestedStackTest(functional_base.FunctionalTestsBase):
+    @test.requires_convergence
+    def test_nested_preemption(self):
+        root_tmpl = _tmpl_with_rsrcs(preempt_root_rsrcs,
+                                     preempt_root_out)
+        files = {
+            preempt_nested_stack_type: _tmpl_with_rsrcs(preempt_nested_rsrcs,
+                                                        preempt_nested_out),
+            preempt_delay_stack_type: _tmpl_with_rsrcs(preempt_delay_rsrcs),
+        }
+        stack_id = self.stack_create(template=root_tmpl, files=files,
+                                     parameters={input_param: 'foo'})
+        delay_stack_uuid = self.get_stack_output(stack_id, 'delay_stack')
+
+        # Start an update that includes a long delay in the second nested stack
+        self.update_stack(stack_id, template=root_tmpl, files=files,
+                          parameters={input_param: 'bar'},
+                          expected_status='UPDATE_IN_PROGRESS')
+        self._wait_for_resource_status(delay_stack_uuid, 'delay_resource',
+                                       'UPDATE_IN_PROGRESS')
+
+        # Update again to check that we preempt update of the first nested
+        # stack. This will delete the second nested stack, after preempting the
+        # update of that stack as well, which will cause the delay resource
+        # within to be cancelled.
+        empty_nest_files = {
+            preempt_nested_stack_type: _tmpl_with_rsrcs({}),
+        }
+        self.update_stack(stack_id, template=root_tmpl, files=empty_nest_files,
+                          parameters={input_param: 'baz'})