openstack
/
heat
Code Issues Proposed changes

Convergence: Pick resource from dead engine worker 

When a engine worker crashes or is restarted, the resources being
provisioned in it remain in IN_PROGRESS state. Next stack update should
pick these resources and work on them. The implementation is to set the
status of resource as FAILED and re-trigger check_resource.

Change-Id: Ib7fd73eadd0127f8fae47881b59388b31131daf4
Closes-Bug: #1501161
This commit is contained in:
Anant Patil 2016-01-06 14:24:01 +05:30
parent 100f9153b7
commit b84417b6ce
2 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
heat/engine/worker.py
View File

@ -182,6 +182,12 @@ class WorkerService(service.Service):
except exception.UpdateInProgress: except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id): if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(resource_data) rpc_data = sync_point.serialize_input_data(resource_data)
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
self._rpc_client.check_resource(cnxt, self._rpc_client.check_resource(cnxt,
rsrc.id, rsrc.id,
current_traversal, current_traversal,

8
heat/tests/engine/test_engine_worker.py
View File

@ -188,9 +188,10 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
@mock.patch.object(worker.WorkerService, '_try_steal_engine_lock') @mock.patch.object(worker.WorkerService, '_try_steal_engine_lock')
@mock.patch.object(stack.Stack, 'time_remaining') @mock.patch.object(stack.Stack, 'time_remaining')
@mock.patch.object(resource.Resource, 'state_set')
def test_is_update_traversal_raise_update_inprogress( def test_is_update_traversal_raise_update_inprogress(
self, tr, mock_tsl, mock_cru, mock_crc, mock_pcr, mock_csc, self, mock_ss, tr, mock_tsl, mock_cru, mock_crc, mock_pcr,
mock_cid): mock_csc, mock_cid):
mock_cru.side_effect = exception.UpdateInProgress mock_cru.side_effect = exception.UpdateInProgress
self.worker.engine_id = 'some-thing-else' self.worker.engine_id = 'some-thing-else'
mock_tsl.return_value = True mock_tsl.return_value = True
@ -202,6 +203,9 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
self.resource.stack.t.id, self.resource.stack.t.id,
{}, self.worker.engine_id, {}, self.worker.engine_id,
mock.ANY) mock.ANY)
mock_ss.assert_called_once_with(self.resource.action,
resource.Resource.FAILED,
mock.ANY)
self.assertFalse(mock_crc.called) self.assertFalse(mock_crc.called)
self.assertFalse(mock_pcr.called) self.assertFalse(mock_pcr.called)
self.assertFalse(mock_csc.called) self.assertFalse(mock_csc.called)