Convergence: Pick resource from dead engine worker
When a engine worker crashes or is restarted, the resources being provisioned in it remain in IN_PROGRESS state. Next stack update should pick these resources and work on them. The implementation is to set the status of resource as FAILED and re-trigger check_resource. Change-Id: Ib7fd73eadd0127f8fae47881b59388b31131daf4 Closes-Bug: #1501161
This commit is contained in:
parent
100f9153b7
commit
b84417b6ce
|
@ -182,6 +182,12 @@ class WorkerService(service.Service):
|
||||||
except exception.UpdateInProgress:
|
except exception.UpdateInProgress:
|
||||||
if self._try_steal_engine_lock(cnxt, rsrc.id):
|
if self._try_steal_engine_lock(cnxt, rsrc.id):
|
||||||
rpc_data = sync_point.serialize_input_data(resource_data)
|
rpc_data = sync_point.serialize_input_data(resource_data)
|
||||||
|
# set the resource state as failed
|
||||||
|
status_reason = ('Worker went down '
|
||||||
|
'during resource %s' % rsrc.action)
|
||||||
|
rsrc.state_set(rsrc.action,
|
||||||
|
rsrc.FAILED,
|
||||||
|
six.text_type(status_reason))
|
||||||
self._rpc_client.check_resource(cnxt,
|
self._rpc_client.check_resource(cnxt,
|
||||||
rsrc.id,
|
rsrc.id,
|
||||||
current_traversal,
|
current_traversal,
|
||||||
|
|
|
@ -188,9 +188,10 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
||||||
|
|
||||||
@mock.patch.object(worker.WorkerService, '_try_steal_engine_lock')
|
@mock.patch.object(worker.WorkerService, '_try_steal_engine_lock')
|
||||||
@mock.patch.object(stack.Stack, 'time_remaining')
|
@mock.patch.object(stack.Stack, 'time_remaining')
|
||||||
|
@mock.patch.object(resource.Resource, 'state_set')
|
||||||
def test_is_update_traversal_raise_update_inprogress(
|
def test_is_update_traversal_raise_update_inprogress(
|
||||||
self, tr, mock_tsl, mock_cru, mock_crc, mock_pcr, mock_csc,
|
self, mock_ss, tr, mock_tsl, mock_cru, mock_crc, mock_pcr,
|
||||||
mock_cid):
|
mock_csc, mock_cid):
|
||||||
mock_cru.side_effect = exception.UpdateInProgress
|
mock_cru.side_effect = exception.UpdateInProgress
|
||||||
self.worker.engine_id = 'some-thing-else'
|
self.worker.engine_id = 'some-thing-else'
|
||||||
mock_tsl.return_value = True
|
mock_tsl.return_value = True
|
||||||
|
@ -202,6 +203,9 @@ class CheckWorkflowUpdateTest(common.HeatTestCase):
|
||||||
self.resource.stack.t.id,
|
self.resource.stack.t.id,
|
||||||
{}, self.worker.engine_id,
|
{}, self.worker.engine_id,
|
||||||
mock.ANY)
|
mock.ANY)
|
||||||
|
mock_ss.assert_called_once_with(self.resource.action,
|
||||||
|
resource.Resource.FAILED,
|
||||||
|
mock.ANY)
|
||||||
self.assertFalse(mock_crc.called)
|
self.assertFalse(mock_crc.called)
|
||||||
self.assertFalse(mock_pcr.called)
|
self.assertFalse(mock_pcr.called)
|
||||||
self.assertFalse(mock_csc.called)
|
self.assertFalse(mock_csc.called)
|
||||||
|
|
Loading…
Reference in New Issue