From 63b5c21c8dc998772cb5933babac0bcf8c9c62eb Mon Sep 17 00:00:00 2001 From: Wenzhi Yu Date: Mon, 18 Jul 2016 15:52:44 +0800 Subject: [PATCH] Rollback bay on update failure There is a rollback mechanism in heat after the stack update failed. There should be a rollback mechanism in magnum after bay update failed. This patch add new microversion 1.3 to add rollback support for Magnum bay, user can enable rollback on bay update failure by specifying microversion 1.3 in header( {'OpenStack-API-Version': 'container-infra 1.3'}) and passing 'rollback=True'(http://XXX/v1/bays/XXX/?rollback=True) when issuing bay update reqeust. Change-Id: Idd02769f98078702404a11dc9f7a3339ce4e22eb Partially-Implements: blueprint bay-rollback-on-update-failure --- magnum/api/controllers/v1/bay.py | 17 ++++++++++- magnum/api/controllers/versions.py | 3 +- magnum/conductor/api.py | 4 +-- magnum/conductor/handlers/bay_conductor.py | 17 +++++++---- magnum/objects/bay.py | 3 +- magnum/objects/fields.py | 10 +++++-- magnum/service/periodic.py | 3 +- .../tests/unit/api/controllers/test_root.py | 2 +- .../tests/unit/api/controllers/v1/test_bay.py | 13 ++++++++- .../conductor/handlers/test_bay_conductor.py | 28 +++++++++++++++++-- .../handlers/test_k8s_bay_conductor.py | 3 +- magnum/tests/unit/objects/test_objects.py | 2 +- magnum/tests/unit/service/test_periodic.py | 26 +++++++++++++---- ...ay-on-update-failure-83e5ff8a7904d5c4.yaml | 7 +++++ 14 files changed, 111 insertions(+), 27 deletions(-) create mode 100644 releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml diff --git a/magnum/api/controllers/v1/bay.py b/magnum/api/controllers/v1/bay.py index 4a42e39fc1..99c84727a8 100644 --- a/magnum/api/controllers/v1/bay.py +++ b/magnum/api/controllers/v1/bay.py @@ -386,7 +386,7 @@ class BaysController(base.Controller): res_bay = pecan.request.rpcapi.bay_update(bay) return Bay.convert_with_links(res_bay) - @base.Controller.api_version("1.2") # noqa + @base.Controller.api_version("1.2", "1.2") # noqa @wsme.validate(types.uuid, [BayPatchType]) @expose.expose(BayID, types.uuid_or_name, body=[BayPatchType], status_code=202) @@ -400,6 +400,21 @@ class BaysController(base.Controller): pecan.request.rpcapi.bay_update_async(bay) return BayID(bay.uuid) + @base.Controller.api_version("1.3") # noqa + @wsme.validate(types.uuid, bool, [BayPatchType]) + @expose.expose(BayID, types.uuid_or_name, bool, body=[BayPatchType], + status_code=202) + def patch(self, bay_ident, rollback=False, patch=None): + """Update an existing bay. + + :param bay_ident: UUID or logical name of a bay. + :param rollback: whether to rollback bay on update failure. + :param patch: a json PATCH document to apply to this bay. + """ + bay = self._patch(bay_ident, patch) + pecan.request.rpcapi.bay_update_async(bay, rollback=rollback) + return BayID(bay.uuid) + def _patch(self, bay_ident, patch): context = pecan.request.context bay = api_utils.get_resource('Bay', bay_ident) diff --git a/magnum/api/controllers/versions.py b/magnum/api/controllers/versions.py index 217286c96c..142e7d7f38 100644 --- a/magnum/api/controllers/versions.py +++ b/magnum/api/controllers/versions.py @@ -28,7 +28,8 @@ from magnum.i18n import _ # Add details of new api versions here: BASE_VER = '1.1' -CURRENT_MAX_VER = '1.2' +CURRENT_MAX_VER = '1.3' +# 1.3 Add bay rollback support # 1.2 Async bay operations support # 1.1 Initial version diff --git a/magnum/conductor/api.py b/magnum/conductor/api.py index 7378053891..a74b757878 100644 --- a/magnum/conductor/api.py +++ b/magnum/conductor/api.py @@ -47,8 +47,8 @@ class API(rpc_service.API): def bay_update(self, bay): return self._call('bay_update', bay=bay) - def bay_update_async(self, bay): - self._cast('bay_update', bay=bay) + def bay_update_async(self, bay, rollback=False): + self._cast('bay_update', bay=bay, rollback=rollback) # CA operations diff --git a/magnum/conductor/handlers/bay_conductor.py b/magnum/conductor/handlers/bay_conductor.py index f1e88ab40d..7e9809d5ef 100644 --- a/magnum/conductor/handlers/bay_conductor.py +++ b/magnum/conductor/handlers/bay_conductor.py @@ -114,7 +114,7 @@ def _create_stack(context, osc, bay, bay_create_timeout): return created_stack -def _update_stack(context, osc, bay, scale_manager=None): +def _update_stack(context, osc, bay, scale_manager=None, rollback=False): template_path, heat_params, env_files = _extract_template_definition( context, bay, scale_manager=scale_manager) @@ -126,7 +126,8 @@ def _update_stack(context, osc, bay, scale_manager=None): 'parameters': heat_params, 'environment_files': environment_files, 'template': template, - 'files': tpl_files + 'files': tpl_files, + 'disable_rollback': not rollback } return osc.heat().stacks.update(bay.stack_id, **fields) @@ -173,7 +174,7 @@ class Handler(object): return bay - def bay_update(self, context, bay): + def bay_update(self, context, bay, rollback=False): LOG.debug('bay_heat bay_update') osc = clients.OpenStackClients(context) @@ -204,7 +205,7 @@ class Handler(object): conductor_utils.notify_about_bay_operation( context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING) - _update_stack(context, osc, bay, manager) + _update_stack(context, osc, bay, manager, rollback) self._poll_and_check(osc, bay) return bay @@ -277,9 +278,11 @@ class HeatPoller(object): bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE, bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE, bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE, + bay_status.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE, bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE, bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE, - bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE + bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE, + bay_status.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE } # poll_and_check is detached and polling long time to check status, # so another user/client can call delete bay/stack. @@ -302,7 +305,9 @@ class HeatPoller(object): if stack.stack_status in (bay_status.CREATE_FAILED, bay_status.DELETE_FAILED, - bay_status.UPDATE_FAILED): + bay_status.UPDATE_FAILED, + bay_status.ROLLBACK_COMPLETE, + bay_status.ROLLBACK_FAILED): self._sync_bay_and_template_status(stack) self._bay_failed(stack) conductor_utils.notify_about_bay_operation( diff --git a/magnum/objects/bay.py b/magnum/objects/bay.py index bde96b284c..74b86dff0b 100644 --- a/magnum/objects/bay.py +++ b/magnum/objects/bay.py @@ -35,7 +35,8 @@ class Bay(base.MagnumPersistentObject, base.MagnumObject, # Version 1.5: Reanme 'registry_trust_id' to 'trust_id' # Add 'trustee_user_name', 'trustee_password', # 'trustee_user_id' field - VERSION = '1.5' + # Version 1.6: Add rollback support for Bay + VERSION = '1.6' dbapi = dbapi.get_instance() diff --git a/magnum/objects/fields.py b/magnum/objects/fields.py index a35eebb3e5..fc3dc35624 100644 --- a/magnum/objects/fields.py +++ b/magnum/objects/fields.py @@ -27,6 +27,8 @@ class BayStatus(fields.Enum): DELETE_COMPLETE = 'DELETE_COMPLETE' RESUME_COMPLETE = 'RESUME_COMPLETE' RESTORE_COMPLETE = 'RESTORE_COMPLETE' + ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS' + ROLLBACK_FAILED = 'ROLLBACK_FAILED' ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE' SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE' CHECK_COMPLETE = 'CHECK_COMPLETE' @@ -35,10 +37,12 @@ class BayStatus(fields.Enum): ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE, UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE, DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE, - RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_COMPLETE, - SNAPSHOT_COMPLETE, CHECK_COMPLETE, ADOPT_COMPLETE) + RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_IN_PROGRESS, + ROLLBACK_FAILED, ROLLBACK_COMPLETE, SNAPSHOT_COMPLETE, + CHECK_COMPLETE, ADOPT_COMPLETE) - STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, DELETE_FAILED) + STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, + DELETE_FAILED, ROLLBACK_FAILED) def __init__(self): super(BayStatus, self).__init__(valid_values=BayStatus.ALL) diff --git a/magnum/service/periodic.py b/magnum/service/periodic.py index b7a80c7866..85af675e87 100644 --- a/magnum/service/periodic.py +++ b/magnum/service/periodic.py @@ -75,7 +75,8 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks): osc = clients.OpenStackClients(ctx) status = [bay_status.CREATE_IN_PROGRESS, bay_status.UPDATE_IN_PROGRESS, - bay_status.DELETE_IN_PROGRESS] + bay_status.DELETE_IN_PROGRESS, + bay_status.ROLLBACK_IN_PROGRESS] filters = {'status': status} bays = objects.Bay.list(ctx, filters=filters) if not bays: diff --git a/magnum/tests/unit/api/controllers/test_root.py b/magnum/tests/unit/api/controllers/test_root.py index 6906d5207d..90118625da 100644 --- a/magnum/tests/unit/api/controllers/test_root.py +++ b/magnum/tests/unit/api/controllers/test_root.py @@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest): [{u'href': u'http://localhost/v1/', u'rel': u'self'}], u'status': u'CURRENT', - u'max_version': u'1.2', + u'max_version': u'1.3', u'min_version': u'1.1'}]} self.v1_expected = { diff --git a/magnum/tests/unit/api/controllers/v1/test_bay.py b/magnum/tests/unit/api/controllers/v1/test_bay.py index 25a5b59e48..31b28559e8 100644 --- a/magnum/tests/unit/api/controllers/v1/test_bay.py +++ b/magnum/tests/unit/api/controllers/v1/test_bay.py @@ -216,7 +216,7 @@ class TestPatch(api_base.FunctionalTest): self.mock_bay_update.side_effect = self._simulate_rpc_bay_update self.addCleanup(p.stop) - def _simulate_rpc_bay_update(self, bay): + def _simulate_rpc_bay_update(self, bay, rollback=False): bay.save() return bay @@ -355,6 +355,17 @@ class TestPatch(api_base.FunctionalTest): self.assertEqual(400, response.status_int) self.assertTrue(response.json['errors']) + @mock.patch.object(rpcapi.API, 'bay_update_async') + def test_update_bay_with_rollback_enabled(self, mock_update): + response = self.patch_json( + '/bays/%s/?rollback=True' % self.bay.name, + [{'path': '/node_count', 'value': 4, + 'op': 'replace'}], + headers={'OpenStack-API-Version': 'container-infra 1.3'}) + + mock_update.assert_called_once_with(mock.ANY, rollback=True) + self.assertEqual(202, response.status_code) + def test_remove_ok(self): response = self.get_json('/bays/%s' % self.bay.uuid) self.assertIsNotNone(response['name']) diff --git a/magnum/tests/unit/conductor/handlers/test_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_bay_conductor.py index ccf92e545b..3385c26807 100644 --- a/magnum/tests/unit/conductor/handlers/test_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_bay_conductor.py @@ -76,7 +76,7 @@ class TestHandler(db_base.DbTestCase): mock_update_stack.assert_called_once_with( self.context, mock_openstack_client, self.bay, - mock_scale_manager.return_value) + mock_scale_manager.return_value, False) bay = objects.Bay.get(self.context, self.bay.uuid) self.assertEqual(2, bay.node_count) @@ -142,7 +142,7 @@ class TestHandler(db_base.DbTestCase): mock_update_stack.assert_called_once_with( self.context, mock_openstack_client, self.bay, - mock_scale_manager.return_value) + mock_scale_manager.return_value, False) bay = objects.Bay.get(self.context, self.bay.uuid) self.assertEqual(2, bay.node_count) @@ -604,6 +604,30 @@ class TestHeatPoller(base.TestCase): self.assertEqual(2, bay.node_count) self.assertEqual(1, poller.attempts) + def test_poll_done_by_rollback_complete(self): + mock_heat_stack, bay, poller = self.setup_poll_test() + + mock_heat_stack.stack_status = bay_status.ROLLBACK_COMPLETE + mock_heat_stack.parameters = {'number_of_minions': 1} + self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check) + + self.assertEqual(2, bay.save.call_count) + self.assertEqual(bay_status.ROLLBACK_COMPLETE, bay.status) + self.assertEqual(1, bay.node_count) + self.assertEqual(1, poller.attempts) + + def test_poll_done_by_rollback_failed(self): + mock_heat_stack, bay, poller = self.setup_poll_test() + + mock_heat_stack.stack_status = bay_status.ROLLBACK_FAILED + mock_heat_stack.parameters = {'number_of_minions': 1} + self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check) + + self.assertEqual(2, bay.save.call_count) + self.assertEqual(bay_status.ROLLBACK_FAILED, bay.status) + self.assertEqual(1, bay.node_count) + self.assertEqual(1, poller.attempts) + def test_poll_destroy(self): mock_heat_stack, bay, poller = self.setup_poll_test() diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py index d86fdfed23..f51c92f9e9 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py @@ -669,7 +669,8 @@ class TestBayConductorWithK8s(base.TestCase): 'parameters': {}, 'template': expected_template_contents, 'files': {}, - 'environment_files': [] + 'environment_files': [], + 'disable_rollback': True } mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id, **expected_args) diff --git a/magnum/tests/unit/objects/test_objects.py b/magnum/tests/unit/objects/test_objects.py index 41f38d2ce5..fe13efc3fd 100644 --- a/magnum/tests/unit/objects/test_objects.py +++ b/magnum/tests/unit/objects/test_objects.py @@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject): # For more information on object version testing, read # http://docs.openstack.org/developer/magnum/objects.html object_data = { - 'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d', + 'Bay': '1.6-2386f79585a6c24bc7960884a4d0ebce', 'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853', 'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2', 'MyObj': '1.0-b43567e512438205e32f4e95ca616697', diff --git a/magnum/tests/unit/service/test_periodic.py b/magnum/tests/unit/service/test_periodic.py index b32c901086..0b6dc41e92 100644 --- a/magnum/tests/unit/service/test_periodic.py +++ b/magnum/tests/unit/service/test_periodic.py @@ -57,11 +57,15 @@ class PeriodicTestCase(base.TestCase): trust_attrs.update({'id': 4, 'stack_id': '44', 'status': bay_status.CREATE_COMPLETE}) bay4 = utils.get_test_bay(**trust_attrs) + trust_attrs.update({'id': 5, 'stack_id': '55', + 'status': bay_status.ROLLBACK_IN_PROGRESS}) + bay5 = utils.get_test_bay(**trust_attrs) self.bay1 = objects.Bay(ctx, **bay1) self.bay2 = objects.Bay(ctx, **bay2) self.bay3 = objects.Bay(ctx, **bay3) self.bay4 = objects.Bay(ctx, **bay4) + self.bay5 = objects.Bay(ctx, **bay5) @mock.patch.object(objects.Bay, 'list') @mock.patch('magnum.common.clients.OpenStackClients') @@ -74,8 +78,10 @@ class PeriodicTestCase(base.TestCase): stack_status_reason='fake_reason_11') stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE, stack_status_reason='fake_reason_33') - mock_heat_client.stacks.list.return_value = [stack1, stack3] - get_stacks = {'11': stack1, '33': stack3} + stack5 = fake_stack(id='55', stack_status=bay_status.ROLLBACK_COMPLETE, + stack_status_reason='fake_reason_55') + mock_heat_client.stacks.list.return_value = [stack1, stack3, stack5] + get_stacks = {'11': stack1, '33': stack3, '55': stack5} def stack_get_sideefect(arg): if arg == '22': @@ -85,7 +91,8 @@ class PeriodicTestCase(base.TestCase): mock_heat_client.stacks.get.side_effect = stack_get_sideefect mock_osc = mock_oscc.return_value mock_osc.heat.return_value = mock_heat_client - mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3] + mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3, + self.bay5] mock_keystone_client = mock.MagicMock() mock_keystone_client.client.project_id = "fake_project" @@ -98,6 +105,8 @@ class PeriodicTestCase(base.TestCase): mock_db_destroy.assert_called_once_with(self.bay2.uuid) self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status) self.assertEqual('fake_reason_33', self.bay3.status_reason) + self.assertEqual(bay_status.ROLLBACK_COMPLETE, self.bay5.status) + self.assertEqual('fake_reason_55', self.bay5.status_reason) @mock.patch.object(objects.Bay, 'list') @mock.patch('magnum.common.clients.OpenStackClients') @@ -136,7 +145,9 @@ class PeriodicTestCase(base.TestCase): stack_status=bay_status.DELETE_IN_PROGRESS) stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_IN_PROGRESS) - get_stacks = {'11': stack1, '22': stack2, '33': stack3} + stack5 = fake_stack(id='55', + stack_status=bay_status.ROLLBACK_IN_PROGRESS) + get_stacks = {'11': stack1, '22': stack2, '33': stack3, '55': stack5} def stack_get_sideefect(arg): if arg == '22': @@ -144,15 +155,18 @@ class PeriodicTestCase(base.TestCase): return get_stacks[arg] mock_heat_client.stacks.get.side_effect = stack_get_sideefect - mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3] + mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3, + stack5] mock_osc = mock_oscc.return_value mock_osc.heat.return_value = mock_heat_client - mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3] + mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3, + self.bay5] periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None) self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status) self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status) self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status) + self.assertEqual(bay_status.ROLLBACK_IN_PROGRESS, self.bay5.status) @mock.patch.object(objects.Bay, 'list') @mock.patch('magnum.common.clients.OpenStackClients') diff --git a/releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml b/releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml new file mode 100644 index 0000000000..3a7f038893 --- /dev/null +++ b/releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml @@ -0,0 +1,7 @@ +--- +features: + - Add Microversion 1.3 to support Magnum bay rollback, + user can enable rollback on bay update failure by + setting 'OpenStack-API-Version' to 'container-infra 1.3' + in request header and passing 'rollback=True' param + in bay update request.