Convergence: Pick resource from dead engine worker

When a engine worker crashes or is restarted, the resources being
provisioned in it remain in IN_PROGRESS state. Next stack update should
pick these resources and work on them. The implementation is to set the
status of resource as FAILED and re-trigger check_resource.

Change-Id: Ib7fd73eadd0127f8fae47881b59388b31131daf4
Closes-Bug: #1501161
changes/15/264115/1
Anant Patil 7 years ago
parent 100f9153b7
commit b84417b6ce
  1. 6
      heat/engine/worker.py
  2. 8
      heat/tests/engine/test_engine_worker.py

@ -182,6 +182,12 @@ class WorkerService(service.Service):
except exception.UpdateInProgress:
if self._try_steal_engine_lock(cnxt, rsrc.id):
rpc_data = sync_point.serialize_input_data(resource_data)
# set the resource state as failed
status_reason = ('Worker went down '
'during resource %s' % rsrc.action)
rsrc.state_set(rsrc.action,
rsrc.FAILED,
six.text_type(status_reason))
self._rpc_client.check_resource(cnxt,
rsrc.id,
current_traversal,

@ -188,9 +188,10 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
@mock.patch.object(worker.WorkerService, '_try_steal_engine_lock')
@mock.patch.object(stack.Stack, 'time_remaining')
@mock.patch.object(resource.Resource, 'state_set')
def test_is_update_traversal_raise_update_inprogress(
self, tr, mock_tsl, mock_cru, mock_crc, mock_pcr, mock_csc,
mock_cid):
self, mock_ss, tr, mock_tsl, mock_cru, mock_crc, mock_pcr,
mock_csc, mock_cid):
mock_cru.side_effect = exception.UpdateInProgress
self.worker.engine_id = 'some-thing-else'
mock_tsl.return_value = True
@ -202,6 +203,9 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
self.resource.stack.t.id,
{}, self.worker.engine_id,
mock.ANY)
mock_ss.assert_called_once_with(self.resource.action,
resource.Resource.FAILED,
mock.ANY)
self.assertFalse(mock_crc.called)
self.assertFalse(mock_pcr.called)
self.assertFalse(mock_csc.called)

Loading…
Cancel
Save