Rollback bay on update failure

There is a rollback mechanism in heat after the stack
update failed. There should be a rollback mechanism in
magnum after bay update failed.

This patch add new microversion 1.3 to add rollback
support for Magnum bay, user can enable rollback on bay
update failure by specifying microversion 1.3 in header(
{'OpenStack-API-Version': 'container-infra 1.3'}) and
passing 'rollback=True'(http://XXX/v1/bays/XXX/?rollback=True)
when issuing bay update reqeust.

Change-Id: Idd02769f98078702404a11dc9f7a3339ce4e22eb
Partially-Implements: blueprint bay-rollback-on-update-failure
This commit is contained in:
Wenzhi Yu 2016-07-18 15:52:44 +08:00
parent 4b66daafd7
commit 63b5c21c8d
14 changed files with 111 additions and 27 deletions

View File

@ -386,7 +386,7 @@ class BaysController(base.Controller):
res_bay = pecan.request.rpcapi.bay_update(bay) res_bay = pecan.request.rpcapi.bay_update(bay)
return Bay.convert_with_links(res_bay) return Bay.convert_with_links(res_bay)
@base.Controller.api_version("1.2") # noqa @base.Controller.api_version("1.2", "1.2") # noqa
@wsme.validate(types.uuid, [BayPatchType]) @wsme.validate(types.uuid, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, body=[BayPatchType], @expose.expose(BayID, types.uuid_or_name, body=[BayPatchType],
status_code=202) status_code=202)
@ -400,6 +400,21 @@ class BaysController(base.Controller):
pecan.request.rpcapi.bay_update_async(bay) pecan.request.rpcapi.bay_update_async(bay)
return BayID(bay.uuid) return BayID(bay.uuid)
@base.Controller.api_version("1.3") # noqa
@wsme.validate(types.uuid, bool, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, bool, body=[BayPatchType],
status_code=202)
def patch(self, bay_ident, rollback=False, patch=None):
"""Update an existing bay.
:param bay_ident: UUID or logical name of a bay.
:param rollback: whether to rollback bay on update failure.
:param patch: a json PATCH document to apply to this bay.
"""
bay = self._patch(bay_ident, patch)
pecan.request.rpcapi.bay_update_async(bay, rollback=rollback)
return BayID(bay.uuid)
def _patch(self, bay_ident, patch): def _patch(self, bay_ident, patch):
context = pecan.request.context context = pecan.request.context
bay = api_utils.get_resource('Bay', bay_ident) bay = api_utils.get_resource('Bay', bay_ident)

View File

@ -28,7 +28,8 @@ from magnum.i18n import _
# Add details of new api versions here: # Add details of new api versions here:
BASE_VER = '1.1' BASE_VER = '1.1'
CURRENT_MAX_VER = '1.2' CURRENT_MAX_VER = '1.3'
# 1.3 Add bay rollback support
# 1.2 Async bay operations support # 1.2 Async bay operations support
# 1.1 Initial version # 1.1 Initial version

View File

@ -47,8 +47,8 @@ class API(rpc_service.API):
def bay_update(self, bay): def bay_update(self, bay):
return self._call('bay_update', bay=bay) return self._call('bay_update', bay=bay)
def bay_update_async(self, bay): def bay_update_async(self, bay, rollback=False):
self._cast('bay_update', bay=bay) self._cast('bay_update', bay=bay, rollback=rollback)
# CA operations # CA operations

View File

@ -114,7 +114,7 @@ def _create_stack(context, osc, bay, bay_create_timeout):
return created_stack return created_stack
def _update_stack(context, osc, bay, scale_manager=None): def _update_stack(context, osc, bay, scale_manager=None, rollback=False):
template_path, heat_params, env_files = _extract_template_definition( template_path, heat_params, env_files = _extract_template_definition(
context, bay, scale_manager=scale_manager) context, bay, scale_manager=scale_manager)
@ -126,7 +126,8 @@ def _update_stack(context, osc, bay, scale_manager=None):
'parameters': heat_params, 'parameters': heat_params,
'environment_files': environment_files, 'environment_files': environment_files,
'template': template, 'template': template,
'files': tpl_files 'files': tpl_files,
'disable_rollback': not rollback
} }
return osc.heat().stacks.update(bay.stack_id, **fields) return osc.heat().stacks.update(bay.stack_id, **fields)
@ -173,7 +174,7 @@ class Handler(object):
return bay return bay
def bay_update(self, context, bay): def bay_update(self, context, bay, rollback=False):
LOG.debug('bay_heat bay_update') LOG.debug('bay_heat bay_update')
osc = clients.OpenStackClients(context) osc = clients.OpenStackClients(context)
@ -204,7 +205,7 @@ class Handler(object):
conductor_utils.notify_about_bay_operation( conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING) context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING)
_update_stack(context, osc, bay, manager) _update_stack(context, osc, bay, manager, rollback)
self._poll_and_check(osc, bay) self._poll_and_check(osc, bay)
return bay return bay
@ -277,9 +278,11 @@ class HeatPoller(object):
bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE, bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE, bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE, bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE, bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE,
bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE, bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE,
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
} }
# poll_and_check is detached and polling long time to check status, # poll_and_check is detached and polling long time to check status,
# so another user/client can call delete bay/stack. # so another user/client can call delete bay/stack.
@ -302,7 +305,9 @@ class HeatPoller(object):
if stack.stack_status in (bay_status.CREATE_FAILED, if stack.stack_status in (bay_status.CREATE_FAILED,
bay_status.DELETE_FAILED, bay_status.DELETE_FAILED,
bay_status.UPDATE_FAILED): bay_status.UPDATE_FAILED,
bay_status.ROLLBACK_COMPLETE,
bay_status.ROLLBACK_FAILED):
self._sync_bay_and_template_status(stack) self._sync_bay_and_template_status(stack)
self._bay_failed(stack) self._bay_failed(stack)
conductor_utils.notify_about_bay_operation( conductor_utils.notify_about_bay_operation(

View File

@ -35,7 +35,8 @@ class Bay(base.MagnumPersistentObject, base.MagnumObject,
# Version 1.5: Reanme 'registry_trust_id' to 'trust_id' # Version 1.5: Reanme 'registry_trust_id' to 'trust_id'
# Add 'trustee_user_name', 'trustee_password', # Add 'trustee_user_name', 'trustee_password',
# 'trustee_user_id' field # 'trustee_user_id' field
VERSION = '1.5' # Version 1.6: Add rollback support for Bay
VERSION = '1.6'
dbapi = dbapi.get_instance() dbapi = dbapi.get_instance()

View File

@ -27,6 +27,8 @@ class BayStatus(fields.Enum):
DELETE_COMPLETE = 'DELETE_COMPLETE' DELETE_COMPLETE = 'DELETE_COMPLETE'
RESUME_COMPLETE = 'RESUME_COMPLETE' RESUME_COMPLETE = 'RESUME_COMPLETE'
RESTORE_COMPLETE = 'RESTORE_COMPLETE' RESTORE_COMPLETE = 'RESTORE_COMPLETE'
ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
ROLLBACK_FAILED = 'ROLLBACK_FAILED'
ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE' ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE' SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE'
CHECK_COMPLETE = 'CHECK_COMPLETE' CHECK_COMPLETE = 'CHECK_COMPLETE'
@ -35,10 +37,12 @@ class BayStatus(fields.Enum):
ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE, ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE,
UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE, UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE,
DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE, DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE,
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_COMPLETE, RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_IN_PROGRESS,
SNAPSHOT_COMPLETE, CHECK_COMPLETE, ADOPT_COMPLETE) ROLLBACK_FAILED, ROLLBACK_COMPLETE, SNAPSHOT_COMPLETE,
CHECK_COMPLETE, ADOPT_COMPLETE)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, DELETE_FAILED) STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED,
DELETE_FAILED, ROLLBACK_FAILED)
def __init__(self): def __init__(self):
super(BayStatus, self).__init__(valid_values=BayStatus.ALL) super(BayStatus, self).__init__(valid_values=BayStatus.ALL)

View File

@ -75,7 +75,8 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
osc = clients.OpenStackClients(ctx) osc = clients.OpenStackClients(ctx)
status = [bay_status.CREATE_IN_PROGRESS, status = [bay_status.CREATE_IN_PROGRESS,
bay_status.UPDATE_IN_PROGRESS, bay_status.UPDATE_IN_PROGRESS,
bay_status.DELETE_IN_PROGRESS] bay_status.DELETE_IN_PROGRESS,
bay_status.ROLLBACK_IN_PROGRESS]
filters = {'status': status} filters = {'status': status}
bays = objects.Bay.list(ctx, filters=filters) bays = objects.Bay.list(ctx, filters=filters)
if not bays: if not bays:

View File

@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
[{u'href': u'http://localhost/v1/', [{u'href': u'http://localhost/v1/',
u'rel': u'self'}], u'rel': u'self'}],
u'status': u'CURRENT', u'status': u'CURRENT',
u'max_version': u'1.2', u'max_version': u'1.3',
u'min_version': u'1.1'}]} u'min_version': u'1.1'}]}
self.v1_expected = { self.v1_expected = {

View File

@ -216,7 +216,7 @@ class TestPatch(api_base.FunctionalTest):
self.mock_bay_update.side_effect = self._simulate_rpc_bay_update self.mock_bay_update.side_effect = self._simulate_rpc_bay_update
self.addCleanup(p.stop) self.addCleanup(p.stop)
def _simulate_rpc_bay_update(self, bay): def _simulate_rpc_bay_update(self, bay, rollback=False):
bay.save() bay.save()
return bay return bay
@ -355,6 +355,17 @@ class TestPatch(api_base.FunctionalTest):
self.assertEqual(400, response.status_int) self.assertEqual(400, response.status_int)
self.assertTrue(response.json['errors']) self.assertTrue(response.json['errors'])
@mock.patch.object(rpcapi.API, 'bay_update_async')
def test_update_bay_with_rollback_enabled(self, mock_update):
response = self.patch_json(
'/bays/%s/?rollback=True' % self.bay.name,
[{'path': '/node_count', 'value': 4,
'op': 'replace'}],
headers={'OpenStack-API-Version': 'container-infra 1.3'})
mock_update.assert_called_once_with(mock.ANY, rollback=True)
self.assertEqual(202, response.status_code)
def test_remove_ok(self): def test_remove_ok(self):
response = self.get_json('/bays/%s' % self.bay.uuid) response = self.get_json('/bays/%s' % self.bay.uuid)
self.assertIsNotNone(response['name']) self.assertIsNotNone(response['name'])

View File

@ -76,7 +76,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with( mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay, self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value) mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid) bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count) self.assertEqual(2, bay.node_count)
@ -142,7 +142,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with( mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay, self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value) mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid) bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count) self.assertEqual(2, bay.node_count)
@ -604,6 +604,30 @@ class TestHeatPoller(base.TestCase):
self.assertEqual(2, bay.node_count) self.assertEqual(2, bay.node_count)
self.assertEqual(1, poller.attempts) self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_complete(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_COMPLETE
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_failed(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_FAILED
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_FAILED, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_destroy(self): def test_poll_destroy(self):
mock_heat_stack, bay, poller = self.setup_poll_test() mock_heat_stack, bay, poller = self.setup_poll_test()

View File

@ -669,7 +669,8 @@ class TestBayConductorWithK8s(base.TestCase):
'parameters': {}, 'parameters': {},
'template': expected_template_contents, 'template': expected_template_contents,
'files': {}, 'files': {},
'environment_files': [] 'environment_files': [],
'disable_rollback': True
} }
mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id, mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id,
**expected_args) **expected_args)

View File

@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read # For more information on object version testing, read
# http://docs.openstack.org/developer/magnum/objects.html # http://docs.openstack.org/developer/magnum/objects.html
object_data = { object_data = {
'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d', 'Bay': '1.6-2386f79585a6c24bc7960884a4d0ebce',
'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853', 'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853',
'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2', 'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2',
'MyObj': '1.0-b43567e512438205e32f4e95ca616697', 'MyObj': '1.0-b43567e512438205e32f4e95ca616697',

View File

@ -57,11 +57,15 @@ class PeriodicTestCase(base.TestCase):
trust_attrs.update({'id': 4, 'stack_id': '44', trust_attrs.update({'id': 4, 'stack_id': '44',
'status': bay_status.CREATE_COMPLETE}) 'status': bay_status.CREATE_COMPLETE})
bay4 = utils.get_test_bay(**trust_attrs) bay4 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 5, 'stack_id': '55',
'status': bay_status.ROLLBACK_IN_PROGRESS})
bay5 = utils.get_test_bay(**trust_attrs)
self.bay1 = objects.Bay(ctx, **bay1) self.bay1 = objects.Bay(ctx, **bay1)
self.bay2 = objects.Bay(ctx, **bay2) self.bay2 = objects.Bay(ctx, **bay2)
self.bay3 = objects.Bay(ctx, **bay3) self.bay3 = objects.Bay(ctx, **bay3)
self.bay4 = objects.Bay(ctx, **bay4) self.bay4 = objects.Bay(ctx, **bay4)
self.bay5 = objects.Bay(ctx, **bay5)
@mock.patch.object(objects.Bay, 'list') @mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.common.clients.OpenStackClients')
@ -74,8 +78,10 @@ class PeriodicTestCase(base.TestCase):
stack_status_reason='fake_reason_11') stack_status_reason='fake_reason_11')
stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE, stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE,
stack_status_reason='fake_reason_33') stack_status_reason='fake_reason_33')
mock_heat_client.stacks.list.return_value = [stack1, stack3] stack5 = fake_stack(id='55', stack_status=bay_status.ROLLBACK_COMPLETE,
get_stacks = {'11': stack1, '33': stack3} stack_status_reason='fake_reason_55')
mock_heat_client.stacks.list.return_value = [stack1, stack3, stack5]
get_stacks = {'11': stack1, '33': stack3, '55': stack5}
def stack_get_sideefect(arg): def stack_get_sideefect(arg):
if arg == '22': if arg == '22':
@ -85,7 +91,8 @@ class PeriodicTestCase(base.TestCase):
mock_heat_client.stacks.get.side_effect = stack_get_sideefect mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_osc = mock_oscc.return_value mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3] mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
mock_keystone_client = mock.MagicMock() mock_keystone_client = mock.MagicMock()
mock_keystone_client.client.project_id = "fake_project" mock_keystone_client.client.project_id = "fake_project"
@ -98,6 +105,8 @@ class PeriodicTestCase(base.TestCase):
mock_db_destroy.assert_called_once_with(self.bay2.uuid) mock_db_destroy.assert_called_once_with(self.bay2.uuid)
self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status) self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status)
self.assertEqual('fake_reason_33', self.bay3.status_reason) self.assertEqual('fake_reason_33', self.bay3.status_reason)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, self.bay5.status)
self.assertEqual('fake_reason_55', self.bay5.status_reason)
@mock.patch.object(objects.Bay, 'list') @mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.common.clients.OpenStackClients')
@ -136,7 +145,9 @@ class PeriodicTestCase(base.TestCase):
stack_status=bay_status.DELETE_IN_PROGRESS) stack_status=bay_status.DELETE_IN_PROGRESS)
stack3 = fake_stack(id='33', stack3 = fake_stack(id='33',
stack_status=bay_status.UPDATE_IN_PROGRESS) stack_status=bay_status.UPDATE_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3} stack5 = fake_stack(id='55',
stack_status=bay_status.ROLLBACK_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3, '55': stack5}
def stack_get_sideefect(arg): def stack_get_sideefect(arg):
if arg == '22': if arg == '22':
@ -144,15 +155,18 @@ class PeriodicTestCase(base.TestCase):
return get_stacks[arg] return get_stacks[arg]
mock_heat_client.stacks.get.side_effect = stack_get_sideefect mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3] mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3,
stack5]
mock_osc = mock_oscc.return_value mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3] mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None) periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status) self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status)
self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status) self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status)
self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status) self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status)
self.assertEqual(bay_status.ROLLBACK_IN_PROGRESS, self.bay5.status)
@mock.patch.object(objects.Bay, 'list') @mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients') @mock.patch('magnum.common.clients.OpenStackClients')

View File

@ -0,0 +1,7 @@
---
features:
- Add Microversion 1.3 to support Magnum bay rollback,
user can enable rollback on bay update failure by
setting 'OpenStack-API-Version' to 'container-infra 1.3'
in request header and passing 'rollback=True' param
in bay update request.