Always do cleanup after a failed stack update
Move the cleanup code (that sets the status to FAILED and merges the new environment into the old one) so that it runs on any exception, including exit exceptions (due to e.g. the thread being cancelled or the engine restarted). In the case of the environment this isn't as good as merging them before the update starts (since it's possible for Heat to die without time to run exception handlers), but at least it leaves things consistent in the orderly case. Change-Id: Ia4d2deca393ac9e250932c5cdb3691b6ee2b3ef4 Partial-bug: #1544348 Related-bug: #1492433
This commit is contained in:
parent
9d47ee0d0b
commit
2c24badd55
|
@ -1322,6 +1322,7 @@ class Stack(collections.Mapping):
|
||||||
existing_params = environment.Environment({env_fmt.PARAMETERS:
|
existing_params = environment.Environment({env_fmt.PARAMETERS:
|
||||||
self.t.env.params})
|
self.t.env.params})
|
||||||
previous_template_id = None
|
previous_template_id = None
|
||||||
|
should_rollback = False
|
||||||
try:
|
try:
|
||||||
update_task = update.StackUpdate(
|
update_task = update.StackUpdate(
|
||||||
self, newstack, backup_stack,
|
self, newstack, backup_stack,
|
||||||
|
@ -1362,14 +1363,17 @@ class Stack(collections.Mapping):
|
||||||
except scheduler.Timeout:
|
except scheduler.Timeout:
|
||||||
self.status = self.FAILED
|
self.status = self.FAILED
|
||||||
self.status_reason = 'Timed out'
|
self.status_reason = 'Timed out'
|
||||||
except (ForcedCancel, exception.ResourceFailure) as e:
|
except (ForcedCancel, Exception) as e:
|
||||||
# If rollback is enabled when resource failure occurred,
|
# If rollback is enabled when resource failure occurred,
|
||||||
# we do another update, with the existing template,
|
# we do another update, with the existing template,
|
||||||
# so we roll back to the original state
|
# so we roll back to the original state
|
||||||
if self._update_exception_handler(
|
should_rollback = self._update_exception_handler(e, action,
|
||||||
exc=e, action=action, update_task=update_task):
|
update_task)
|
||||||
|
if should_rollback:
|
||||||
yield self.update_task(oldstack, action=self.ROLLBACK)
|
yield self.update_task(oldstack, action=self.ROLLBACK)
|
||||||
return
|
except BaseException as e:
|
||||||
|
with excutils.save_and_reraise_exception():
|
||||||
|
self._update_exception_handler(e, action, update_task)
|
||||||
else:
|
else:
|
||||||
LOG.debug('Deleting backup stack')
|
LOG.debug('Deleting backup stack')
|
||||||
backup_stack.delete(backup=True)
|
backup_stack.delete(backup=True)
|
||||||
|
@ -1379,29 +1383,33 @@ class Stack(collections.Mapping):
|
||||||
self.t = newstack.t
|
self.t = newstack.t
|
||||||
template_outputs = self.t[self.t.OUTPUTS]
|
template_outputs = self.t[self.t.OUTPUTS]
|
||||||
self.outputs = self.resolve_static_data(template_outputs)
|
self.outputs = self.resolve_static_data(template_outputs)
|
||||||
|
finally:
|
||||||
|
if should_rollback:
|
||||||
|
# Already handled in rollback task
|
||||||
|
return
|
||||||
|
|
||||||
# Don't use state_set to do only one update query and avoid race
|
# Don't use state_set to do only one update query and avoid race
|
||||||
# condition with the COMPLETE status
|
# condition with the COMPLETE status
|
||||||
self.action = action
|
self.action = action
|
||||||
|
|
||||||
self._send_notification_and_add_event()
|
self._send_notification_and_add_event()
|
||||||
if self.status == self.FAILED:
|
if self.status == self.FAILED:
|
||||||
# Since template was incrementally updated based on existing and
|
# Since template was incrementally updated based on existing
|
||||||
# new stack resources, we should have user params of both.
|
# and new stack resources, we should have user params of both.
|
||||||
existing_params.load(newstack.t.env.user_env_as_dict())
|
existing_params.load(newstack.t.env.user_env_as_dict())
|
||||||
self.t.env = existing_params
|
self.t.env = existing_params
|
||||||
self.t.store(self.context)
|
self.t.store(self.context)
|
||||||
backup_stack.t.env = existing_params
|
backup_stack.t.env = existing_params
|
||||||
backup_stack.t.store(self.context)
|
backup_stack.t.store(self.context)
|
||||||
self.store()
|
self.store()
|
||||||
|
|
||||||
if previous_template_id is not None:
|
if previous_template_id is not None:
|
||||||
raw_template_object.RawTemplate.delete(self.context,
|
raw_template_object.RawTemplate.delete(self.context,
|
||||||
previous_template_id)
|
previous_template_id)
|
||||||
|
|
||||||
lifecycle_plugin_utils.do_post_ops(self.context, self,
|
lifecycle_plugin_utils.do_post_ops(self.context, self,
|
||||||
newstack, action,
|
newstack, action,
|
||||||
(self.status == self.FAILED))
|
(self.status == self.FAILED))
|
||||||
|
|
||||||
def _update_exception_handler(self, exc, action, update_task):
|
def _update_exception_handler(self, exc, action, update_task):
|
||||||
"""Handle exceptions in update_task.
|
"""Handle exceptions in update_task.
|
||||||
|
@ -1419,8 +1427,10 @@ class Stack(collections.Mapping):
|
||||||
if isinstance(exc, ForcedCancel):
|
if isinstance(exc, ForcedCancel):
|
||||||
update_task.updater.cancel_all()
|
update_task.updater.cancel_all()
|
||||||
return exc.with_rollback or not self.disable_rollback
|
return exc.with_rollback or not self.disable_rollback
|
||||||
|
elif isinstance(exc, exception.ResourceFailure):
|
||||||
return not self.disable_rollback
|
return not self.disable_rollback
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
def _message_parser(self, message):
|
def _message_parser(self, message):
|
||||||
if message == rpc_api.THREAD_CANCEL:
|
if message == rpc_api.THREAD_CANCEL:
|
||||||
|
|
Loading…
Reference in New Issue