openstack
/
heat
Code Issues Proposed changes

Protect prepare_update_replace() with resource lock 

When we consolidated resource locking with resource state transitions in
a7376f7494, prepare_update_replace() moved
outside of the locked section, so that other update operations could
potentially conflict with it. This change ensures that we call it while the
lock is still held (if we have transitioned the resource to
UPDATE_IN_PROGRESS, and thus acquired the lock), or that we acquire the
lock before calling it (if UpdateReplace is raised before attempting to
update the resource).

To avoid unnecessary locking and unlocking, we only take the lock if the
Resource plugin has overridden the default handler method (either
restore_prev_rsrc() or prepare_for_replace()), since the defaults do
nothing.

Change-Id: Ie32836ed643e440143bde8b83aeb4d6159acd034
Closes-Bug: #1727127
This commit is contained in:
Zane Bitter 2017-10-30 17:07:12 -04:00 committed by rabi
parent 93b4551d9a
commit b0e18270b4
2 changed files with 78 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
heat/engine/resource.py
View File

@ -1481,16 +1481,34 @@ class Resource(status.ResourceStatus):
"error: %s", ex) "error: %s", ex)
return after_props, before_props return after_props, before_props
def _prepare_update_replace_handler(self, action):
"""Return the handler method for preparing to replace a resource.
This may be either restore_prev_rsrc() (in the case of a legacy
rollback) or, more typically, prepare_for_replace().
If the plugin has not overridden the method, then None is returned in
place of the default method (which is empty anyway).
"""
if (self.stack.action == 'ROLLBACK' and
self.stack.status == 'IN_PROGRESS' and
not self.stack.convergence):
# handle case, when it's rollback and we should restore
# old resource
if self.restore_prev_rsrc != Resource.restore_prev_rsrc:
return self.restore_prev_rsrc
else:
if self.prepare_for_replace != Resource.prepare_for_replace:
return self.prepare_for_replace
return None
def _prepare_update_replace(self, action): def _prepare_update_replace(self, action):
handler = self._prepare_update_replace_handler(action)
if handler is None:
return
try: try:
if (self.stack.action == 'ROLLBACK' and handler()
self.stack.status == 'IN_PROGRESS' and
not self.stack.convergence):
# handle case, when it's rollback and we should restore
# old resource
self.restore_prev_rsrc()
else:
self.prepare_for_replace()
except Exception as e: except Exception as e:
# if any exception happen, we should set the resource to # if any exception happen, we should set the resource to
# FAILED, then raise ResourceFailure # FAILED, then raise ResourceFailure
@ -1549,61 +1567,65 @@ class Resource(status.ResourceStatus):
needs_update = self._needs_update(after, before, needs_update = self._needs_update(after, before,
after_props, before_props, after_props, before_props,
prev_resource) prev_resource)
if not needs_update: except UpdateReplace:
if update_templ_func is not None: with excutils.save_and_reraise_exception():
update_templ_func(persist=True) if self._prepare_update_replace_handler(action) is not None:
if self.status == self.FAILED: with self.lock(self._calling_engine_id):
status_reason = _('Update status to COMPLETE for ' self._prepare_update_replace(action)
'FAILED resource neither update ' except exception.ResourceActionRestricted as ae:
'nor replace.') failure = exception.ResourceFailure(ae, self, action)
lock = (self.LOCK_RESPECT if self.stack.convergence self._add_event(action, self.FAILED, six.text_type(ae))
else self.LOCK_NONE) raise failure
self.state_set(self.action, self.COMPLETE,
status_reason, lock=lock)
return
if not self.stack.convergence: if not needs_update:
if (self.action, self.status) in ( if update_templ_func is not None:
(self.CREATE, self.IN_PROGRESS), update_templ_func(persist=True)
(self.UPDATE, self.IN_PROGRESS), if self.status == self.FAILED:
(self.ADOPT, self.IN_PROGRESS)): status_reason = _('Update status to COMPLETE for '
exc = Exception(_('Resource update already requested')) 'FAILED resource neither update '
raise exception.ResourceFailure(exc, self, action) 'nor replace.')
lock = (self.LOCK_RESPECT if self.stack.convergence
else self.LOCK_NONE)
self.state_set(self.action, self.COMPLETE,
status_reason, lock=lock)
return
LOG.info('updating %s', self) if not self.stack.convergence:
if (self.action, self.status) in (
(self.CREATE, self.IN_PROGRESS),
(self.UPDATE, self.IN_PROGRESS),
(self.ADOPT, self.IN_PROGRESS)):
exc = Exception(_('Resource update already requested'))
raise exception.ResourceFailure(exc, self, action)
self.updated_time = datetime.utcnow() LOG.info('updating %s', self)
with self._action_recorder(action, UpdateReplace): self.updated_time = datetime.utcnow()
after_props.validate()
tmpl_diff = self.update_template_diff(after.freeze(), before) with self._action_recorder(action, UpdateReplace):
after_props.validate()
self.properties = before_props
tmpl_diff = self.update_template_diff(after.freeze(), before)
try:
if tmpl_diff and self.needs_replace_with_tmpl_diff(tmpl_diff): if tmpl_diff and self.needs_replace_with_tmpl_diff(tmpl_diff):
raise UpdateReplace(self) raise UpdateReplace(self)
prop_diff = self.update_template_diff_properties(after_props, prop_diff = self.update_template_diff_properties(after_props,
before_props) before_props)
self.properties = before_props
yield self.action_handler_task(action, yield self.action_handler_task(action,
args=[after, tmpl_diff, args=[after, tmpl_diff,
prop_diff]) prop_diff])
self.t = after except UpdateReplace:
self.reparse() with excutils.save_and_reraise_exception():
self._update_stored_properties() self._prepare_update_replace(action)
if update_templ_func is not None:
# template/requires will be persisted by _action_recorder()
update_templ_func(persist=False)
except exception.ResourceActionRestricted as ae: self.t = after
# catch all ResourceActionRestricted exceptions self.reparse()
failure = exception.ResourceFailure(ae, self, action) self._update_stored_properties()
self._add_event(action, self.FAILED, six.text_type(ae)) if update_templ_func is not None:
raise failure # template/requires will be persisted by _action_recorder()
except UpdateReplace: update_templ_func(persist=False)
# catch all UpdateReplace exceptions
self._prepare_update_replace(action)
raise
yield self._break_if_required( yield self._break_if_required(
self.UPDATE, environment.HOOK_POST_UPDATE) self.UPDATE, environment.HOOK_POST_UPDATE)
@ -2066,15 +2088,18 @@ class Resource(status.ResourceStatus):
def lock(self, engine_id): def lock(self, engine_id):
self._calling_engine_id = engine_id self._calling_engine_id = engine_id
try: try:
self._store_with_lock({}, self.LOCK_ACQUIRE) if engine_id is not None:
self._store_with_lock({}, self.LOCK_ACQUIRE)
yield yield
except exception.UpdateInProgress: except exception.UpdateInProgress:
raise raise
except BaseException: except BaseException:
with excutils.save_and_reraise_exception(): with excutils.save_and_reraise_exception():
self._store_with_lock({}, self.LOCK_RELEASE) if engine_id is not None:
self._store_with_lock({}, self.LOCK_RELEASE)
else: else:
self._store_with_lock({}, self.LOCK_RELEASE) if engine_id is not None:
self._store_with_lock({}, self.LOCK_RELEASE)
def _resolve_any_attribute(self, attr): def _resolve_any_attribute(self, attr):
"""Method for resolving any attribute, including base attributes. """Method for resolving any attribute, including base attributes.

2
heat/tests/test_resource.py
View File

@ -2409,7 +2409,7 @@ class ResourceTest(common.HeatTestCase):
self.assertEqual(stack.t.id, res.current_template_id) self.assertEqual(stack.t.id, res.current_template_id)
# ensure that requires was not updated # ensure that requires was not updated
self.assertItemsEqual([2], res.requires) self.assertItemsEqual([2], res.requires)
self._assert_resource_lock(res.id, None, None) self._assert_resource_lock(res.id, None, 2)
def test_convergence_update_replace_rollback(self): def test_convergence_update_replace_rollback(self):
rsrc_def = rsrc_defn.ResourceDefinition('test_res', rsrc_def = rsrc_defn.ResourceDefinition('test_res',