openstack
/
magnum
Code Issues Proposed changes

Merge "Rollback bay on update failure"

changes/24/355724/1
Jenkins 2016-08-16 04:59:35 +00:00 committed by Gerrit Code Review
parent 02a2378ca0 63b5c21c8d
commit 5372cd92b4
14 changed files with 111 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
magnum/api/controllers/v1/bay.py
View File

@ -386,7 +386,7 @@ class BaysController(base.Controller):
res_bay = pecan.request.rpcapi.bay_update(bay)
return Bay.convert_with_links(res_bay)
@base.Controller.api_version("1.2") # noqa
@base.Controller.api_version("1.2", "1.2") # noqa
@wsme.validate(types.uuid, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, body=[BayPatchType],
status_code=202)
@ -400,6 +400,21 @@ class BaysController(base.Controller):
pecan.request.rpcapi.bay_update_async(bay)
return BayID(bay.uuid)
@base.Controller.api_version("1.3") # noqa
@wsme.validate(types.uuid, bool, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, bool, body=[BayPatchType],
status_code=202)
def patch(self, bay_ident, rollback=False, patch=None):
"""Update an existing bay.
:param bay_ident: UUID or logical name of a bay.
:param rollback: whether to rollback bay on update failure.
:param patch: a json PATCH document to apply to this bay.
"""
bay = self._patch(bay_ident, patch)
pecan.request.rpcapi.bay_update_async(bay, rollback=rollback)
return BayID(bay.uuid)
def _patch(self, bay_ident, patch):
context = pecan.request.context
bay = api_utils.get_resource('Bay', bay_ident)

3
magnum/api/controllers/versions.py
View File

@ -28,7 +28,8 @@ from magnum.i18n import _
# Add details of new api versions here:
BASE_VER = '1.1'
CURRENT_MAX_VER = '1.2'
CURRENT_MAX_VER = '1.3'
# 1.3 Add bay rollback support
# 1.2 Async bay operations support
# 1.1 Initial version

4
magnum/conductor/api.py
View File

@ -47,8 +47,8 @@ class API(rpc_service.API):
def bay_update(self, bay):
return self._call('bay_update', bay=bay)
def bay_update_async(self, bay):
self._cast('bay_update', bay=bay)
def bay_update_async(self, bay, rollback=False):
self._cast('bay_update', bay=bay, rollback=rollback)
# CA operations

17
magnum/conductor/handlers/bay_conductor.py
View File

@ -114,7 +114,7 @@ def _create_stack(context, osc, bay, bay_create_timeout):
return created_stack
def _update_stack(context, osc, bay, scale_manager=None):
def _update_stack(context, osc, bay, scale_manager=None, rollback=False):
template_path, heat_params, env_files = _extract_template_definition(
context, bay, scale_manager=scale_manager)
@ -126,7 +126,8 @@ def _update_stack(context, osc, bay, scale_manager=None):
'parameters': heat_params,
'environment_files': environment_files,
'template': template,
'files': tpl_files
'files': tpl_files,
'disable_rollback': not rollback
}
return osc.heat().stacks.update(bay.stack_id, **fields)
@ -173,7 +174,7 @@ class Handler(object):
return bay
def bay_update(self, context, bay):
def bay_update(self, context, bay, rollback=False):
LOG.debug('bay_heat bay_update')
osc = clients.OpenStackClients(context)
@ -204,7 +205,7 @@ class Handler(object):
conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING)
_update_stack(context, osc, bay, manager)
_update_stack(context, osc, bay, manager, rollback)
self._poll_and_check(osc, bay)
return bay
@ -280,9 +281,11 @@ class HeatPoller(object):
bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE,
bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE,
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
}
# poll_and_check is detached and polling long time to check status,
# so another user/client can call delete bay/stack.
@ -305,7 +308,9 @@ class HeatPoller(object):
if stack.stack_status in (bay_status.CREATE_FAILED,
bay_status.DELETE_FAILED,
bay_status.UPDATE_FAILED):
bay_status.UPDATE_FAILED,
bay_status.ROLLBACK_COMPLETE,
bay_status.ROLLBACK_FAILED):
self._sync_bay_and_template_status(stack)
self._bay_failed(stack)
conductor_utils.notify_about_bay_operation(

3
magnum/objects/bay.py
View File

@ -35,7 +35,8 @@ class Bay(base.MagnumPersistentObject, base.MagnumObject,
# Version 1.5: Reanme 'registry_trust_id' to 'trust_id'
# Add 'trustee_user_name', 'trustee_password',
# 'trustee_user_id' field
VERSION = '1.5'
# Version 1.6: Add rollback support for Bay
VERSION = '1.6'
dbapi = dbapi.get_instance()

10
magnum/objects/fields.py
View File

@ -27,6 +27,8 @@ class BayStatus(fields.Enum):
DELETE_COMPLETE = 'DELETE_COMPLETE'
RESUME_COMPLETE = 'RESUME_COMPLETE'
RESTORE_COMPLETE = 'RESTORE_COMPLETE'
ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
ROLLBACK_FAILED = 'ROLLBACK_FAILED'
ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE'
CHECK_COMPLETE = 'CHECK_COMPLETE'
@ -35,10 +37,12 @@ class BayStatus(fields.Enum):
ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE,
UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE,
DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE,
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_COMPLETE,
SNAPSHOT_COMPLETE, CHECK_COMPLETE, ADOPT_COMPLETE)
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_IN_PROGRESS,
ROLLBACK_FAILED, ROLLBACK_COMPLETE, SNAPSHOT_COMPLETE,
CHECK_COMPLETE, ADOPT_COMPLETE)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, DELETE_FAILED)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED,
DELETE_FAILED, ROLLBACK_FAILED)
def __init__(self):
super(BayStatus, self).__init__(valid_values=BayStatus.ALL)

3
magnum/service/periodic.py
View File

@ -75,7 +75,8 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
osc = clients.OpenStackClients(ctx)
status = [bay_status.CREATE_IN_PROGRESS,
bay_status.UPDATE_IN_PROGRESS,
bay_status.DELETE_IN_PROGRESS]
bay_status.DELETE_IN_PROGRESS,
bay_status.ROLLBACK_IN_PROGRESS]
filters = {'status': status}
bays = objects.Bay.list(ctx, filters=filters)
if not bays:

2
magnum/tests/unit/api/controllers/test_root.py
View File

@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
[{u'href': u'http://localhost/v1/',
u'rel': u'self'}],
u'status': u'CURRENT',
u'max_version': u'1.2',
u'max_version': u'1.3',
u'min_version': u'1.1'}]}
self.v1_expected = {

13
magnum/tests/unit/api/controllers/v1/test_bay.py
View File

@ -216,7 +216,7 @@ class TestPatch(api_base.FunctionalTest):
self.mock_bay_update.side_effect = self._simulate_rpc_bay_update
self.addCleanup(p.stop)
def _simulate_rpc_bay_update(self, bay):
def _simulate_rpc_bay_update(self, bay, rollback=False):
bay.save()
return bay
@ -355,6 +355,17 @@ class TestPatch(api_base.FunctionalTest):
self.assertEqual(400, response.status_int)
self.assertTrue(response.json['errors'])
@mock.patch.object(rpcapi.API, 'bay_update_async')
def test_update_bay_with_rollback_enabled(self, mock_update):
response = self.patch_json(
'/bays/%s/?rollback=True' % self.bay.name,
[{'path': '/node_count', 'value': 4,
'op': 'replace'}],
headers={'OpenStack-API-Version': 'container-infra 1.3'})
mock_update.assert_called_once_with(mock.ANY, rollback=True)
self.assertEqual(202, response.status_code)
def test_remove_ok(self):
response = self.get_json('/bays/%s' % self.bay.uuid)
self.assertIsNotNone(response['name'])

28
magnum/tests/unit/conductor/handlers/test_bay_conductor.py
View File

@ -76,7 +76,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value)
mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count)
@ -142,7 +142,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value)
mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count)
@ -604,6 +604,30 @@ class TestHeatPoller(base.TestCase):
self.assertEqual(2, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_complete(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_COMPLETE
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_failed(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_FAILED
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_FAILED, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_destroy(self):
mock_heat_stack, bay, poller = self.setup_poll_test()

3
magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py
View File

@ -669,7 +669,8 @@ class TestBayConductorWithK8s(base.TestCase):
'parameters': {},
'template': expected_template_contents,
'files': {},
'environment_files': []
'environment_files': [],
'disable_rollback': True
}
mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id,
**expected_args)

2
magnum/tests/unit/objects/test_objects.py
View File

@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read
# http://docs.openstack.org/developer/magnum/objects.html
object_data = {
'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d',
'Bay': '1.6-2386f79585a6c24bc7960884a4d0ebce',
'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853',
'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2',
'MyObj': '1.0-b43567e512438205e32f4e95ca616697',

26
magnum/tests/unit/service/test_periodic.py
View File

@ -57,11 +57,15 @@ class PeriodicTestCase(base.TestCase):
trust_attrs.update({'id': 4, 'stack_id': '44',
'status': bay_status.CREATE_COMPLETE})
bay4 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 5, 'stack_id': '55',
'status': bay_status.ROLLBACK_IN_PROGRESS})
bay5 = utils.get_test_bay(**trust_attrs)
self.bay1 = objects.Bay(ctx, **bay1)
self.bay2 = objects.Bay(ctx, **bay2)
self.bay3 = objects.Bay(ctx, **bay3)
self.bay4 = objects.Bay(ctx, **bay4)
self.bay5 = objects.Bay(ctx, **bay5)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
@ -74,8 +78,10 @@ class PeriodicTestCase(base.TestCase):
stack_status_reason='fake_reason_11')
stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE,
stack_status_reason='fake_reason_33')
mock_heat_client.stacks.list.return_value = [stack1, stack3]
get_stacks = {'11': stack1, '33': stack3}
stack5 = fake_stack(id='55', stack_status=bay_status.ROLLBACK_COMPLETE,
stack_status_reason='fake_reason_55')
mock_heat_client.stacks.list.return_value = [stack1, stack3, stack5]
get_stacks = {'11': stack1, '33': stack3, '55': stack5}
def stack_get_sideefect(arg):
if arg == '22':
@ -85,7 +91,8 @@ class PeriodicTestCase(base.TestCase):
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
mock_keystone_client = mock.MagicMock()
mock_keystone_client.client.project_id = "fake_project"
@ -98,6 +105,8 @@ class PeriodicTestCase(base.TestCase):
mock_db_destroy.assert_called_once_with(self.bay2.uuid)
self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status)
self.assertEqual('fake_reason_33', self.bay3.status_reason)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, self.bay5.status)
self.assertEqual('fake_reason_55', self.bay5.status_reason)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
@ -136,7 +145,9 @@ class PeriodicTestCase(base.TestCase):
stack_status=bay_status.DELETE_IN_PROGRESS)
stack3 = fake_stack(id='33',
stack_status=bay_status.UPDATE_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3}
stack5 = fake_stack(id='55',
stack_status=bay_status.ROLLBACK_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3, '55': stack5}
def stack_get_sideefect(arg):
if arg == '22':
@ -144,15 +155,18 @@ class PeriodicTestCase(base.TestCase):
return get_stacks[arg]
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3]
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3,
stack5]
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status)
self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status)
self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status)
self.assertEqual(bay_status.ROLLBACK_IN_PROGRESS, self.bay5.status)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')

7
releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml Normal file
View File

@ -0,0 +1,7 @@
---
features:
- Add Microversion 1.3 to support Magnum bay rollback,
user can enable rollback on bay update failure by
setting 'OpenStack-API-Version' to 'container-infra 1.3'
in request header and passing 'rollback=True' param
in bay update request.