From b84417b6cea85229f78a8b371ee6eb7e00e3411c Mon Sep 17 00:00:00 2001 From: Anant Patil Date: Wed, 6 Jan 2016 14:24:01 +0530 Subject: [PATCH] Convergence: Pick resource from dead engine worker When a engine worker crashes or is restarted, the resources being provisioned in it remain in IN_PROGRESS state. Next stack update should pick these resources and work on them. The implementation is to set the status of resource as FAILED and re-trigger check_resource. Change-Id: Ib7fd73eadd0127f8fae47881b59388b31131daf4 Closes-Bug: #1501161 --- heat/engine/worker.py | 6 ++++++ heat/tests/engine/test_engine_worker.py | 8 ++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/heat/engine/worker.py b/heat/engine/worker.py index a406918f5d..4ead4ef8db 100644 --- a/heat/engine/worker.py +++ b/heat/engine/worker.py @@ -182,6 +182,12 @@ class WorkerService(service.Service): except exception.UpdateInProgress: if self._try_steal_engine_lock(cnxt, rsrc.id): rpc_data = sync_point.serialize_input_data(resource_data) + # set the resource state as failed + status_reason = ('Worker went down ' + 'during resource %s' % rsrc.action) + rsrc.state_set(rsrc.action, + rsrc.FAILED, + six.text_type(status_reason)) self._rpc_client.check_resource(cnxt, rsrc.id, current_traversal, diff --git a/heat/tests/engine/test_engine_worker.py b/heat/tests/engine/test_engine_worker.py index 56fb8661a8..e3243cb878 100644 --- a/heat/tests/engine/test_engine_worker.py +++ b/heat/tests/engine/test_engine_worker.py @@ -188,9 +188,10 @@ class CheckWorkflowUpdateTest(common.HeatTestCase): @mock.patch.object(worker.WorkerService, '_try_steal_engine_lock') @mock.patch.object(stack.Stack, 'time_remaining') + @mock.patch.object(resource.Resource, 'state_set') def test_is_update_traversal_raise_update_inprogress( - self, tr, mock_tsl, mock_cru, mock_crc, mock_pcr, mock_csc, - mock_cid): + self, mock_ss, tr, mock_tsl, mock_cru, mock_crc, mock_pcr, + mock_csc, mock_cid): mock_cru.side_effect = exception.UpdateInProgress self.worker.engine_id = 'some-thing-else' mock_tsl.return_value = True @@ -202,6 +203,9 @@ class CheckWorkflowUpdateTest(common.HeatTestCase): self.resource.stack.t.id, {}, self.worker.engine_id, mock.ANY) + mock_ss.assert_called_once_with(self.resource.action, + resource.Resource.FAILED, + mock.ANY) self.assertFalse(mock_crc.called) self.assertFalse(mock_pcr.called) self.assertFalse(mock_csc.called)