Browse Source

Rollback bay on update failure

There is a rollback mechanism in heat after the stack
update failed. There should be a rollback mechanism in
magnum after bay update failed.

This patch add new microversion 1.3 to add rollback
support for Magnum bay, user can enable rollback on bay
update failure by specifying microversion 1.3 in header(
{'OpenStack-API-Version': 'container-infra 1.3'}) and
passing 'rollback=True'(http://XXX/v1/bays/XXX/?rollback=True)
when issuing bay update reqeust.

Change-Id: Idd02769f98078702404a11dc9f7a3339ce4e22eb
Partially-Implements: blueprint bay-rollback-on-update-failure
changes/78/343478/11
Wenzhi Yu 6 years ago
parent
commit
63b5c21c8d
  1. 17
      magnum/api/controllers/v1/bay.py
  2. 3
      magnum/api/controllers/versions.py
  3. 4
      magnum/conductor/api.py
  4. 17
      magnum/conductor/handlers/bay_conductor.py
  5. 3
      magnum/objects/bay.py
  6. 10
      magnum/objects/fields.py
  7. 3
      magnum/service/periodic.py
  8. 2
      magnum/tests/unit/api/controllers/test_root.py
  9. 13
      magnum/tests/unit/api/controllers/v1/test_bay.py
  10. 28
      magnum/tests/unit/conductor/handlers/test_bay_conductor.py
  11. 3
      magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py
  12. 2
      magnum/tests/unit/objects/test_objects.py
  13. 26
      magnum/tests/unit/service/test_periodic.py
  14. 7
      releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml

17
magnum/api/controllers/v1/bay.py

@ -386,7 +386,7 @@ class BaysController(base.Controller):
res_bay = pecan.request.rpcapi.bay_update(bay)
return Bay.convert_with_links(res_bay)
@base.Controller.api_version("1.2") # noqa
@base.Controller.api_version("1.2", "1.2") # noqa
@wsme.validate(types.uuid, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, body=[BayPatchType],
status_code=202)
@ -400,6 +400,21 @@ class BaysController(base.Controller):
pecan.request.rpcapi.bay_update_async(bay)
return BayID(bay.uuid)
@base.Controller.api_version("1.3") # noqa
@wsme.validate(types.uuid, bool, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, bool, body=[BayPatchType],
status_code=202)
def patch(self, bay_ident, rollback=False, patch=None):
"""Update an existing bay.
:param bay_ident: UUID or logical name of a bay.
:param rollback: whether to rollback bay on update failure.
:param patch: a json PATCH document to apply to this bay.
"""
bay = self._patch(bay_ident, patch)
pecan.request.rpcapi.bay_update_async(bay, rollback=rollback)
return BayID(bay.uuid)
def _patch(self, bay_ident, patch):
context = pecan.request.context
bay = api_utils.get_resource('Bay', bay_ident)

3
magnum/api/controllers/versions.py

@ -28,7 +28,8 @@ from magnum.i18n import _
# Add details of new api versions here:
BASE_VER = '1.1'
CURRENT_MAX_VER = '1.2'
CURRENT_MAX_VER = '1.3'
# 1.3 Add bay rollback support
# 1.2 Async bay operations support
# 1.1 Initial version

4
magnum/conductor/api.py

@ -47,8 +47,8 @@ class API(rpc_service.API):
def bay_update(self, bay):
return self._call('bay_update', bay=bay)
def bay_update_async(self, bay):
self._cast('bay_update', bay=bay)
def bay_update_async(self, bay, rollback=False):
self._cast('bay_update', bay=bay, rollback=rollback)
# CA operations

17
magnum/conductor/handlers/bay_conductor.py

@ -114,7 +114,7 @@ def _create_stack(context, osc, bay, bay_create_timeout):
return created_stack
def _update_stack(context, osc, bay, scale_manager=None):
def _update_stack(context, osc, bay, scale_manager=None, rollback=False):
template_path, heat_params, env_files = _extract_template_definition(
context, bay, scale_manager=scale_manager)
@ -126,7 +126,8 @@ def _update_stack(context, osc, bay, scale_manager=None):
'parameters': heat_params,
'environment_files': environment_files,
'template': template,
'files': tpl_files
'files': tpl_files,
'disable_rollback': not rollback
}
return osc.heat().stacks.update(bay.stack_id, **fields)
@ -173,7 +174,7 @@ class Handler(object):
return bay
def bay_update(self, context, bay):
def bay_update(self, context, bay, rollback=False):
LOG.debug('bay_heat bay_update')
osc = clients.OpenStackClients(context)
@ -204,7 +205,7 @@ class Handler(object):
conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING)
_update_stack(context, osc, bay, manager)
_update_stack(context, osc, bay, manager, rollback)
self._poll_and_check(osc, bay)
return bay
@ -277,9 +278,11 @@ class HeatPoller(object):
bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE,
bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE,
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
}
# poll_and_check is detached and polling long time to check status,
# so another user/client can call delete bay/stack.
@ -302,7 +305,9 @@ class HeatPoller(object):
if stack.stack_status in (bay_status.CREATE_FAILED,
bay_status.DELETE_FAILED,
bay_status.UPDATE_FAILED):
bay_status.UPDATE_FAILED,
bay_status.ROLLBACK_COMPLETE,
bay_status.ROLLBACK_FAILED):
self._sync_bay_and_template_status(stack)
self._bay_failed(stack)
conductor_utils.notify_about_bay_operation(

3
magnum/objects/bay.py

@ -35,7 +35,8 @@ class Bay(base.MagnumPersistentObject, base.MagnumObject,
# Version 1.5: Reanme 'registry_trust_id' to 'trust_id'
# Add 'trustee_user_name', 'trustee_password',
# 'trustee_user_id' field
VERSION = '1.5'
# Version 1.6: Add rollback support for Bay
VERSION = '1.6'
dbapi = dbapi.get_instance()

10
magnum/objects/fields.py

@ -27,6 +27,8 @@ class BayStatus(fields.Enum):
DELETE_COMPLETE = 'DELETE_COMPLETE'
RESUME_COMPLETE = 'RESUME_COMPLETE'
RESTORE_COMPLETE = 'RESTORE_COMPLETE'
ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
ROLLBACK_FAILED = 'ROLLBACK_FAILED'
ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE'
CHECK_COMPLETE = 'CHECK_COMPLETE'
@ -35,10 +37,12 @@ class BayStatus(fields.Enum):
ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE,
UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE,
DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE,
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_COMPLETE,
SNAPSHOT_COMPLETE, CHECK_COMPLETE, ADOPT_COMPLETE)
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_IN_PROGRESS,
ROLLBACK_FAILED, ROLLBACK_COMPLETE, SNAPSHOT_COMPLETE,
CHECK_COMPLETE, ADOPT_COMPLETE)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, DELETE_FAILED)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED,
DELETE_FAILED, ROLLBACK_FAILED)
def __init__(self):
super(BayStatus, self).__init__(valid_values=BayStatus.ALL)

3
magnum/service/periodic.py

@ -75,7 +75,8 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
osc = clients.OpenStackClients(ctx)
status = [bay_status.CREATE_IN_PROGRESS,
bay_status.UPDATE_IN_PROGRESS,
bay_status.DELETE_IN_PROGRESS]
bay_status.DELETE_IN_PROGRESS,
bay_status.ROLLBACK_IN_PROGRESS]
filters = {'status': status}
bays = objects.Bay.list(ctx, filters=filters)
if not bays:

2
magnum/tests/unit/api/controllers/test_root.py

@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
[{u'href': u'http://localhost/v1/',
u'rel': u'self'}],
u'status': u'CURRENT',
u'max_version': u'1.2',
u'max_version': u'1.3',
u'min_version': u'1.1'}]}
self.v1_expected = {

13
magnum/tests/unit/api/controllers/v1/test_bay.py

@ -216,7 +216,7 @@ class TestPatch(api_base.FunctionalTest):
self.mock_bay_update.side_effect = self._simulate_rpc_bay_update
self.addCleanup(p.stop)
def _simulate_rpc_bay_update(self, bay):
def _simulate_rpc_bay_update(self, bay, rollback=False):
bay.save()
return bay
@ -355,6 +355,17 @@ class TestPatch(api_base.FunctionalTest):
self.assertEqual(400, response.status_int)
self.assertTrue(response.json['errors'])
@mock.patch.object(rpcapi.API, 'bay_update_async')
def test_update_bay_with_rollback_enabled(self, mock_update):
response = self.patch_json(
'/bays/%s/?rollback=True' % self.bay.name,
[{'path': '/node_count', 'value': 4,
'op': 'replace'}],
headers={'OpenStack-API-Version': 'container-infra 1.3'})
mock_update.assert_called_once_with(mock.ANY, rollback=True)
self.assertEqual(202, response.status_code)
def test_remove_ok(self):
response = self.get_json('/bays/%s' % self.bay.uuid)
self.assertIsNotNone(response['name'])

28
magnum/tests/unit/conductor/handlers/test_bay_conductor.py

@ -76,7 +76,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value)
mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count)
@ -142,7 +142,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value)
mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count)
@ -604,6 +604,30 @@ class TestHeatPoller(base.TestCase):
self.assertEqual(2, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_complete(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_COMPLETE
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_failed(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_FAILED
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_FAILED, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_destroy(self):
mock_heat_stack, bay, poller = self.setup_poll_test()

3
magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py

@ -669,7 +669,8 @@ class TestBayConductorWithK8s(base.TestCase):
'parameters': {},
'template': expected_template_contents,
'files': {},
'environment_files': []
'environment_files': [],
'disable_rollback': True
}
mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id,
**expected_args)

2
magnum/tests/unit/objects/test_objects.py

@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read
# http://docs.openstack.org/developer/magnum/objects.html
object_data = {
'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d',
'Bay': '1.6-2386f79585a6c24bc7960884a4d0ebce',
'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853',
'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2',
'MyObj': '1.0-b43567e512438205e32f4e95ca616697',

26
magnum/tests/unit/service/test_periodic.py

@ -57,11 +57,15 @@ class PeriodicTestCase(base.TestCase):
trust_attrs.update({'id': 4, 'stack_id': '44',
'status': bay_status.CREATE_COMPLETE})
bay4 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 5, 'stack_id': '55',
'status': bay_status.ROLLBACK_IN_PROGRESS})
bay5 = utils.get_test_bay(**trust_attrs)
self.bay1 = objects.Bay(ctx, **bay1)
self.bay2 = objects.Bay(ctx, **bay2)
self.bay3 = objects.Bay(ctx, **bay3)
self.bay4 = objects.Bay(ctx, **bay4)
self.bay5 = objects.Bay(ctx, **bay5)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
@ -74,8 +78,10 @@ class PeriodicTestCase(base.TestCase):
stack_status_reason='fake_reason_11')
stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE,
stack_status_reason='fake_reason_33')
mock_heat_client.stacks.list.return_value = [stack1, stack3]
get_stacks = {'11': stack1, '33': stack3}
stack5 = fake_stack(id='55', stack_status=bay_status.ROLLBACK_COMPLETE,
stack_status_reason='fake_reason_55')
mock_heat_client.stacks.list.return_value = [stack1, stack3, stack5]
get_stacks = {'11': stack1, '33': stack3, '55': stack5}
def stack_get_sideefect(arg):
if arg == '22':
@ -85,7 +91,8 @@ class PeriodicTestCase(base.TestCase):
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
mock_keystone_client = mock.MagicMock()
mock_keystone_client.client.project_id = "fake_project"
@ -98,6 +105,8 @@ class PeriodicTestCase(base.TestCase):
mock_db_destroy.assert_called_once_with(self.bay2.uuid)
self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status)
self.assertEqual('fake_reason_33', self.bay3.status_reason)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, self.bay5.status)
self.assertEqual('fake_reason_55', self.bay5.status_reason)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
@ -136,7 +145,9 @@ class PeriodicTestCase(base.TestCase):
stack_status=bay_status.DELETE_IN_PROGRESS)
stack3 = fake_stack(id='33',
stack_status=bay_status.UPDATE_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3}
stack5 = fake_stack(id='55',
stack_status=bay_status.ROLLBACK_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3, '55': stack5}
def stack_get_sideefect(arg):
if arg == '22':
@ -144,15 +155,18 @@ class PeriodicTestCase(base.TestCase):
return get_stacks[arg]
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3]
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3,
stack5]
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status)
self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status)
self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status)
self.assertEqual(bay_status.ROLLBACK_IN_PROGRESS, self.bay5.status)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')

7
releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml

@ -0,0 +1,7 @@
---
features:
- Add Microversion 1.3 to support Magnum bay rollback,
user can enable rollback on bay update failure by
setting 'OpenStack-API-Version' to 'container-infra 1.3'
in request header and passing 'rollback=True' param
in bay update request.
Loading…
Cancel
Save