Merge "Rollback bay on update failure"

changes/24/355724/1
Jenkins 6 years ago committed by Gerrit Code Review
commit 5372cd92b4
  1. 17
      magnum/api/controllers/v1/bay.py
  2. 3
      magnum/api/controllers/versions.py
  3. 4
      magnum/conductor/api.py
  4. 17
      magnum/conductor/handlers/bay_conductor.py
  5. 3
      magnum/objects/bay.py
  6. 10
      magnum/objects/fields.py
  7. 3
      magnum/service/periodic.py
  8. 2
      magnum/tests/unit/api/controllers/test_root.py
  9. 13
      magnum/tests/unit/api/controllers/v1/test_bay.py
  10. 28
      magnum/tests/unit/conductor/handlers/test_bay_conductor.py
  11. 3
      magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py
  12. 2
      magnum/tests/unit/objects/test_objects.py
  13. 26
      magnum/tests/unit/service/test_periodic.py
  14. 7
      releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml

@ -386,7 +386,7 @@ class BaysController(base.Controller):
res_bay = pecan.request.rpcapi.bay_update(bay)
return Bay.convert_with_links(res_bay)
@base.Controller.api_version("1.2") # noqa
@base.Controller.api_version("1.2", "1.2") # noqa
@wsme.validate(types.uuid, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, body=[BayPatchType],
status_code=202)
@ -400,6 +400,21 @@ class BaysController(base.Controller):
pecan.request.rpcapi.bay_update_async(bay)
return BayID(bay.uuid)
@base.Controller.api_version("1.3") # noqa
@wsme.validate(types.uuid, bool, [BayPatchType])
@expose.expose(BayID, types.uuid_or_name, bool, body=[BayPatchType],
status_code=202)
def patch(self, bay_ident, rollback=False, patch=None):
"""Update an existing bay.
:param bay_ident: UUID or logical name of a bay.
:param rollback: whether to rollback bay on update failure.
:param patch: a json PATCH document to apply to this bay.
"""
bay = self._patch(bay_ident, patch)
pecan.request.rpcapi.bay_update_async(bay, rollback=rollback)
return BayID(bay.uuid)
def _patch(self, bay_ident, patch):
context = pecan.request.context
bay = api_utils.get_resource('Bay', bay_ident)

@ -28,7 +28,8 @@ from magnum.i18n import _
# Add details of new api versions here:
BASE_VER = '1.1'
CURRENT_MAX_VER = '1.2'
CURRENT_MAX_VER = '1.3'
# 1.3 Add bay rollback support
# 1.2 Async bay operations support
# 1.1 Initial version

@ -47,8 +47,8 @@ class API(rpc_service.API):
def bay_update(self, bay):
return self._call('bay_update', bay=bay)
def bay_update_async(self, bay):
self._cast('bay_update', bay=bay)
def bay_update_async(self, bay, rollback=False):
self._cast('bay_update', bay=bay, rollback=rollback)
# CA operations

@ -114,7 +114,7 @@ def _create_stack(context, osc, bay, bay_create_timeout):
return created_stack
def _update_stack(context, osc, bay, scale_manager=None):
def _update_stack(context, osc, bay, scale_manager=None, rollback=False):
template_path, heat_params, env_files = _extract_template_definition(
context, bay, scale_manager=scale_manager)
@ -126,7 +126,8 @@ def _update_stack(context, osc, bay, scale_manager=None):
'parameters': heat_params,
'environment_files': environment_files,
'template': template,
'files': tpl_files
'files': tpl_files,
'disable_rollback': not rollback
}
return osc.heat().stacks.update(bay.stack_id, **fields)
@ -173,7 +174,7 @@ class Handler(object):
return bay
def bay_update(self, context, bay):
def bay_update(self, context, bay, rollback=False):
LOG.debug('bay_heat bay_update')
osc = clients.OpenStackClients(context)
@ -204,7 +205,7 @@ class Handler(object):
conductor_utils.notify_about_bay_operation(
context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING)
_update_stack(context, osc, bay, manager)
_update_stack(context, osc, bay, manager, rollback)
self._poll_and_check(osc, bay)
return bay
@ -280,9 +281,11 @@ class HeatPoller(object):
bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE,
bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE,
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE
bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
bay_status.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
}
# poll_and_check is detached and polling long time to check status,
# so another user/client can call delete bay/stack.
@ -305,7 +308,9 @@ class HeatPoller(object):
if stack.stack_status in (bay_status.CREATE_FAILED,
bay_status.DELETE_FAILED,
bay_status.UPDATE_FAILED):
bay_status.UPDATE_FAILED,
bay_status.ROLLBACK_COMPLETE,
bay_status.ROLLBACK_FAILED):
self._sync_bay_and_template_status(stack)
self._bay_failed(stack)
conductor_utils.notify_about_bay_operation(

@ -35,7 +35,8 @@ class Bay(base.MagnumPersistentObject, base.MagnumObject,
# Version 1.5: Reanme 'registry_trust_id' to 'trust_id'
# Add 'trustee_user_name', 'trustee_password',
# 'trustee_user_id' field
VERSION = '1.5'
# Version 1.6: Add rollback support for Bay
VERSION = '1.6'
dbapi = dbapi.get_instance()

@ -27,6 +27,8 @@ class BayStatus(fields.Enum):
DELETE_COMPLETE = 'DELETE_COMPLETE'
RESUME_COMPLETE = 'RESUME_COMPLETE'
RESTORE_COMPLETE = 'RESTORE_COMPLETE'
ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
ROLLBACK_FAILED = 'ROLLBACK_FAILED'
ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE'
CHECK_COMPLETE = 'CHECK_COMPLETE'
@ -35,10 +37,12 @@ class BayStatus(fields.Enum):
ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE,
UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE,
DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE,
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_COMPLETE,
SNAPSHOT_COMPLETE, CHECK_COMPLETE, ADOPT_COMPLETE)
RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_IN_PROGRESS,
ROLLBACK_FAILED, ROLLBACK_COMPLETE, SNAPSHOT_COMPLETE,
CHECK_COMPLETE, ADOPT_COMPLETE)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, DELETE_FAILED)
STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED,
DELETE_FAILED, ROLLBACK_FAILED)
def __init__(self):
super(BayStatus, self).__init__(valid_values=BayStatus.ALL)

@ -75,7 +75,8 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
osc = clients.OpenStackClients(ctx)
status = [bay_status.CREATE_IN_PROGRESS,
bay_status.UPDATE_IN_PROGRESS,
bay_status.DELETE_IN_PROGRESS]
bay_status.DELETE_IN_PROGRESS,
bay_status.ROLLBACK_IN_PROGRESS]
filters = {'status': status}
bays = objects.Bay.list(ctx, filters=filters)
if not bays:

@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
[{u'href': u'http://localhost/v1/',
u'rel': u'self'}],
u'status': u'CURRENT',
u'max_version': u'1.2',
u'max_version': u'1.3',
u'min_version': u'1.1'}]}
self.v1_expected = {

@ -216,7 +216,7 @@ class TestPatch(api_base.FunctionalTest):
self.mock_bay_update.side_effect = self._simulate_rpc_bay_update
self.addCleanup(p.stop)
def _simulate_rpc_bay_update(self, bay):
def _simulate_rpc_bay_update(self, bay, rollback=False):
bay.save()
return bay
@ -355,6 +355,17 @@ class TestPatch(api_base.FunctionalTest):
self.assertEqual(400, response.status_int)
self.assertTrue(response.json['errors'])
@mock.patch.object(rpcapi.API, 'bay_update_async')
def test_update_bay_with_rollback_enabled(self, mock_update):
response = self.patch_json(
'/bays/%s/?rollback=True' % self.bay.name,
[{'path': '/node_count', 'value': 4,
'op': 'replace'}],
headers={'OpenStack-API-Version': 'container-infra 1.3'})
mock_update.assert_called_once_with(mock.ANY, rollback=True)
self.assertEqual(202, response.status_code)
def test_remove_ok(self):
response = self.get_json('/bays/%s' % self.bay.uuid)
self.assertIsNotNone(response['name'])

@ -76,7 +76,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value)
mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count)
@ -142,7 +142,7 @@ class TestHandler(db_base.DbTestCase):
mock_update_stack.assert_called_once_with(
self.context, mock_openstack_client, self.bay,
mock_scale_manager.return_value)
mock_scale_manager.return_value, False)
bay = objects.Bay.get(self.context, self.bay.uuid)
self.assertEqual(2, bay.node_count)
@ -604,6 +604,30 @@ class TestHeatPoller(base.TestCase):
self.assertEqual(2, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_complete(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_COMPLETE
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_done_by_rollback_failed(self):
mock_heat_stack, bay, poller = self.setup_poll_test()
mock_heat_stack.stack_status = bay_status.ROLLBACK_FAILED
mock_heat_stack.parameters = {'number_of_minions': 1}
self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
self.assertEqual(2, bay.save.call_count)
self.assertEqual(bay_status.ROLLBACK_FAILED, bay.status)
self.assertEqual(1, bay.node_count)
self.assertEqual(1, poller.attempts)
def test_poll_destroy(self):
mock_heat_stack, bay, poller = self.setup_poll_test()

@ -669,7 +669,8 @@ class TestBayConductorWithK8s(base.TestCase):
'parameters': {},
'template': expected_template_contents,
'files': {},
'environment_files': []
'environment_files': [],
'disable_rollback': True
}
mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id,
**expected_args)

@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read
# http://docs.openstack.org/developer/magnum/objects.html
object_data = {
'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d',
'Bay': '1.6-2386f79585a6c24bc7960884a4d0ebce',
'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853',
'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2',
'MyObj': '1.0-b43567e512438205e32f4e95ca616697',

@ -57,11 +57,15 @@ class PeriodicTestCase(base.TestCase):
trust_attrs.update({'id': 4, 'stack_id': '44',
'status': bay_status.CREATE_COMPLETE})
bay4 = utils.get_test_bay(**trust_attrs)
trust_attrs.update({'id': 5, 'stack_id': '55',
'status': bay_status.ROLLBACK_IN_PROGRESS})
bay5 = utils.get_test_bay(**trust_attrs)
self.bay1 = objects.Bay(ctx, **bay1)
self.bay2 = objects.Bay(ctx, **bay2)
self.bay3 = objects.Bay(ctx, **bay3)
self.bay4 = objects.Bay(ctx, **bay4)
self.bay5 = objects.Bay(ctx, **bay5)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
@ -74,8 +78,10 @@ class PeriodicTestCase(base.TestCase):
stack_status_reason='fake_reason_11')
stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE,
stack_status_reason='fake_reason_33')
mock_heat_client.stacks.list.return_value = [stack1, stack3]
get_stacks = {'11': stack1, '33': stack3}
stack5 = fake_stack(id='55', stack_status=bay_status.ROLLBACK_COMPLETE,
stack_status_reason='fake_reason_55')
mock_heat_client.stacks.list.return_value = [stack1, stack3, stack5]
get_stacks = {'11': stack1, '33': stack3, '55': stack5}
def stack_get_sideefect(arg):
if arg == '22':
@ -85,7 +91,8 @@ class PeriodicTestCase(base.TestCase):
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
mock_keystone_client = mock.MagicMock()
mock_keystone_client.client.project_id = "fake_project"
@ -98,6 +105,8 @@ class PeriodicTestCase(base.TestCase):
mock_db_destroy.assert_called_once_with(self.bay2.uuid)
self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status)
self.assertEqual('fake_reason_33', self.bay3.status_reason)
self.assertEqual(bay_status.ROLLBACK_COMPLETE, self.bay5.status)
self.assertEqual('fake_reason_55', self.bay5.status_reason)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')
@ -136,7 +145,9 @@ class PeriodicTestCase(base.TestCase):
stack_status=bay_status.DELETE_IN_PROGRESS)
stack3 = fake_stack(id='33',
stack_status=bay_status.UPDATE_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3}
stack5 = fake_stack(id='55',
stack_status=bay_status.ROLLBACK_IN_PROGRESS)
get_stacks = {'11': stack1, '22': stack2, '33': stack3, '55': stack5}
def stack_get_sideefect(arg):
if arg == '22':
@ -144,15 +155,18 @@ class PeriodicTestCase(base.TestCase):
return get_stacks[arg]
mock_heat_client.stacks.get.side_effect = stack_get_sideefect
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3]
mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3,
stack5]
mock_osc = mock_oscc.return_value
mock_osc.heat.return_value = mock_heat_client
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
self.bay5]
periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status)
self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status)
self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status)
self.assertEqual(bay_status.ROLLBACK_IN_PROGRESS, self.bay5.status)
@mock.patch.object(objects.Bay, 'list')
@mock.patch('magnum.common.clients.OpenStackClients')

@ -0,0 +1,7 @@
---
features:
- Add Microversion 1.3 to support Magnum bay rollback,
user can enable rollback on bay update failure by
setting 'OpenStack-API-Version' to 'container-infra 1.3'
in request header and passing 'rollback=True' param
in bay update request.
Loading…
Cancel
Save