openstack/heat
Code Issues Proposed changes

Pre-empt in-progress nested stack updates on new update

Browse Source 
If the parent resource of a nested stack is locked due to an IN_PROGRESS
update, cancel the nested stack update (which will result in the parent
resource being marked FAILED and releasing the lock so that the new
traversal can begin acting on it). This also cancels all descendants of
the nested stack.

This means that a concurrent update no longer gets blocked at a nested
stack boundary until the previous update has finished.

Change-Id: I5f14453ebab75d89672c6eea12de46d48a5147f3
Task: 17760
This commit is contained in:
Zane Bitter 2019-10-23 18:00:58 -04:00
parent 9f3ea0a63e
commit 8cd6a06736
4 changed files with 112 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
heat/engine/check_resource.py
View File

@ -163,6 +163,8 @@ class CheckResource(object):
return True
except exception.UpdateInProgress:
LOG.debug('Waiting for existing update to unlock resource %s',
rsrc.id)
if self._stale_resource_needs_retry(cnxt, rsrc, prev_template_id):
rpc_data = sync_point.serialize_input_data(self.input_data)
self._rpc_client.check_resource(cnxt,
@ -170,6 +172,8 @@ class CheckResource(object):
current_traversal,
rpc_data, is_update,
adopt_stack_data)
else:
rsrc.handle_preempt()
except exception.ResourceFailure as ex:
action = ex.action or rsrc.action
reason = 'Resource %s failed: %s' % (action,

17
heat/engine/resource.py
View File

@ -1456,6 +1456,23 @@ class Resource(status.ResourceStatus):
new_requires=new_requires)
runner(timeout=timeout, progress_callback=progress_callback)
def handle_preempt(self):
"""Pre-empt an in-progress update when a new update is available.
This method is called when a previous convergence update is in
progress but a new update for the resource is available. By default
it does nothing, but subclasses may override it to cancel the
in-progress update if it is safe to do so.
Note that this method does not run in the context of the in-progress
update and has no access to runtime information about it; nor is it
safe to make changes to the Resource in the database. If implemented,
this method should cause the existing update to complete by external
means. If this leaves the resource in a FAILED state, that should be
taken into account in needs_replace_failed().
"""
return
def preview_update(self, after, before, after_props, before_props,
prev_resource, check_init_complete=False):
"""Simulates update without actually updating the resource.

9
heat/engine/resources/stack_resource.py
View File

@ -561,9 +561,10 @@ class StackResource(resource.Resource):
return self._check_status_complete(target_action,
cookie=cookie)
def handle_update_cancel(self, cookie):
def _handle_cancel(self):
stack_identity = self.nested_identifier()
if stack_identity is not None:
LOG.debug('Cancelling %s of %s' % (self.action, self))
try:
self.rpc_client().stack_cancel_update(
self.context,
@ -573,6 +574,12 @@ class StackResource(resource.Resource):
LOG.debug('Nested stack %s not in cancellable state',
stack_identity.stack_name)
def handle_preempt(self):
self._handle_cancel()
def handle_update_cancel(self, cookie):
self._handle_cancel()
def handle_create_cancel(self, cookie):
return self.handle_update_cancel(cookie)

83
heat_integrationtests/functional/test_simultaneous_update.py
View File

@ -12,6 +12,7 @@
import copy
import json
import time
from heat_integrationtests.common import test
@ -91,3 +92,85 @@ class SimultaneousUpdateStackTest(functional_base.FunctionalTestsBase):
time.sleep(50)
self.update_stack(stack_id, after)
input_param = 'input'
preempt_nested_stack_type = 'preempt.yaml'
preempt_root_rsrcs = {
'nested_stack': {
'type': preempt_nested_stack_type,
'properties': {
'input': {'get_param': input_param},
},
}
}
preempt_root_out = {'get_attr': ['nested_stack', 'delay_stack']}
preempt_delay_stack_type = 'delay.yaml'
preempt_nested_rsrcs = {
'delay_stack': {
'type': preempt_delay_stack_type,
'properties': {
'input': {'get_param': input_param},
},
}
}
preempt_nested_out = {'get_resource': 'delay_stack'}
preempt_delay_rsrcs = {
'delay_resource': {
'type': 'OS::Heat::TestResource',
'properties': {
'action_wait_secs': {
'update': 6000,
},
'value': {'get_param': input_param},
},
}
}
def _tmpl_with_rsrcs(rsrcs, output_value=None):
tmpl = {
'heat_template_version': 'queens',
'parameters': {
input_param: {
'type': 'string',
},
},
'resources': rsrcs,
}
if output_value is not None:
outputs = {'delay_stack': {'value': output_value}}
tmpl['outputs'] = outputs
return json.dumps(tmpl)
class SimultaneousUpdateNestedStackTest(functional_base.FunctionalTestsBase):
@test.requires_convergence
def test_nested_preemption(self):
root_tmpl = _tmpl_with_rsrcs(preempt_root_rsrcs,
preempt_root_out)
files = {
preempt_nested_stack_type: _tmpl_with_rsrcs(preempt_nested_rsrcs,
preempt_nested_out),
preempt_delay_stack_type: _tmpl_with_rsrcs(preempt_delay_rsrcs),
}
stack_id = self.stack_create(template=root_tmpl, files=files,
parameters={input_param: 'foo'})
delay_stack_uuid = self.get_stack_output(stack_id, 'delay_stack')
# Start an update that includes a long delay in the second nested stack
self.update_stack(stack_id, template=root_tmpl, files=files,
parameters={input_param: 'bar'},
expected_status='UPDATE_IN_PROGRESS')
self._wait_for_resource_status(delay_stack_uuid, 'delay_resource',
'UPDATE_IN_PROGRESS')
# Update again to check that we preempt update of the first nested
# stack. This will delete the second nested stack, after preempting the
# update of that stack as well, which will cause the delay resource
# within to be cancelled.
empty_nest_files = {
preempt_nested_stack_type: _tmpl_with_rsrcs({}),
}
self.update_stack(stack_id, template=root_tmpl, files=empty_nest_files,
parameters={input_param: 'baz'})