Rollback bay on update failure

There is a rollback mechanism in heat after the stack
update failed. There should be a rollback mechanism in
magnum after bay update failed.

This patch add new microversion 1.3 to add rollback
support for Magnum bay, user can enable rollback on bay
update failure by specifying microversion 1.3 in header(
{'OpenStack-API-Version': 'container-infra 1.3'}) and
passing 'rollback=True'(http://XXX/v1/bays/XXX/?rollback=True)
when issuing bay update reqeust.

Change-Id: Idd02769f98078702404a11dc9f7a3339ce4e22eb
Partially-Implements: blueprint bay-rollback-on-update-failure

magnum/api/controllers/v1/bay.py

@@ -386,7 +386,7 @@ class BaysController(base.Controller):
         res_bay = pecan.request.rpcapi.bay_update(bay)
         return Bay.convert_with_links(res_bay)
 
-    @base.Controller.api_version("1.2")   # noqa
+    @base.Controller.api_version("1.2", "1.2")   # noqa
     @wsme.validate(types.uuid, [BayPatchType])
     @expose.expose(BayID, types.uuid_or_name, body=[BayPatchType],
                    status_code=202)
@@ -400,6 +400,21 @@ class BaysController(base.Controller):
         pecan.request.rpcapi.bay_update_async(bay)
         return BayID(bay.uuid)
 
+    @base.Controller.api_version("1.3")   # noqa
+    @wsme.validate(types.uuid, bool, [BayPatchType])
+    @expose.expose(BayID, types.uuid_or_name, bool, body=[BayPatchType],
+                   status_code=202)
+    def patch(self, bay_ident, rollback=False, patch=None):
+        """Update an existing bay.
+
+        :param bay_ident: UUID or logical name of a bay.
+        :param rollback: whether to rollback bay on update failure.
+        :param patch: a json PATCH document to apply to this bay.
+        """
+        bay = self._patch(bay_ident, patch)
+        pecan.request.rpcapi.bay_update_async(bay, rollback=rollback)
+        return BayID(bay.uuid)
+
     def _patch(self, bay_ident, patch):
         context = pecan.request.context
         bay = api_utils.get_resource('Bay', bay_ident)

magnum/api/controllers/versions.py

@@ -28,7 +28,8 @@ from magnum.i18n import _
 #             Add details of new api versions here:
 
 BASE_VER = '1.1'
-CURRENT_MAX_VER = '1.2'
+CURRENT_MAX_VER = '1.3'
+# 1.3 Add bay rollback support
 # 1.2 Async bay operations support
 # 1.1 Initial version
 

magnum/conductor/api.py

@@ -47,8 +47,8 @@ class API(rpc_service.API):
     def bay_update(self, bay):
         return self._call('bay_update', bay=bay)
 
-    def bay_update_async(self, bay):
+    def bay_update_async(self, bay, rollback=False):
-        self._cast('bay_update', bay=bay)
+        self._cast('bay_update', bay=bay, rollback=rollback)
 
     # CA operations
 

magnum/conductor/handlers/bay_conductor.py

@@ -114,7 +114,7 @@ def _create_stack(context, osc, bay, bay_create_timeout):
     return created_stack
 
 
-def _update_stack(context, osc, bay, scale_manager=None):
+def _update_stack(context, osc, bay, scale_manager=None, rollback=False):
     template_path, heat_params, env_files = _extract_template_definition(
         context, bay, scale_manager=scale_manager)
 
@@ -126,7 +126,8 @@ def _update_stack(context, osc, bay, scale_manager=None):
         'parameters': heat_params,
         'environment_files': environment_files,
         'template': template,
-        'files': tpl_files
+        'files': tpl_files,
+        'disable_rollback': not rollback
     }
 
     return osc.heat().stacks.update(bay.stack_id, **fields)
@@ -173,7 +174,7 @@ class Handler(object):
 
         return bay
 
-    def bay_update(self, context, bay):
+    def bay_update(self, context, bay, rollback=False):
         LOG.debug('bay_heat bay_update')
 
         osc = clients.OpenStackClients(context)
@@ -204,7 +205,7 @@ class Handler(object):
         conductor_utils.notify_about_bay_operation(
             context, taxonomy.ACTION_UPDATE, taxonomy.OUTCOME_PENDING)
 
-        _update_stack(context, osc, bay, manager)
+        _update_stack(context, osc, bay, manager, rollback)
         self._poll_and_check(osc, bay)
 
         return bay
@@ -277,9 +278,11 @@ class HeatPoller(object):
             bay_status.DELETE_COMPLETE: taxonomy.ACTION_DELETE,
             bay_status.CREATE_COMPLETE: taxonomy.ACTION_CREATE,
             bay_status.UPDATE_COMPLETE: taxonomy.ACTION_UPDATE,
+            bay_status.ROLLBACK_COMPLETE: taxonomy.ACTION_UPDATE,
             bay_status.CREATE_FAILED: taxonomy.ACTION_CREATE,
             bay_status.DELETE_FAILED: taxonomy.ACTION_DELETE,
-            bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE
+            bay_status.UPDATE_FAILED: taxonomy.ACTION_UPDATE,
+            bay_status.ROLLBACK_FAILED: taxonomy.ACTION_UPDATE
         }
         # poll_and_check is detached and polling long time to check status,
         # so another user/client can call delete bay/stack.
@@ -302,7 +305,9 @@ class HeatPoller(object):
 
         if stack.stack_status in (bay_status.CREATE_FAILED,
                                   bay_status.DELETE_FAILED,
-                                  bay_status.UPDATE_FAILED):
+                                  bay_status.UPDATE_FAILED,
+                                  bay_status.ROLLBACK_COMPLETE,
+                                  bay_status.ROLLBACK_FAILED):
             self._sync_bay_and_template_status(stack)
             self._bay_failed(stack)
             conductor_utils.notify_about_bay_operation(

magnum/objects/bay.py

@@ -35,7 +35,8 @@ class Bay(base.MagnumPersistentObject, base.MagnumObject,
     # Version 1.5: Reanme 'registry_trust_id' to 'trust_id'
     #              Add 'trustee_user_name', 'trustee_password',
     #              'trustee_user_id' field
-    VERSION = '1.5'
+    # Version 1.6: Add rollback support for Bay
+    VERSION = '1.6'
 
     dbapi = dbapi.get_instance()
 

magnum/objects/fields.py

@@ -27,6 +27,8 @@ class BayStatus(fields.Enum):
     DELETE_COMPLETE = 'DELETE_COMPLETE'
     RESUME_COMPLETE = 'RESUME_COMPLETE'
     RESTORE_COMPLETE = 'RESTORE_COMPLETE'
+    ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
+    ROLLBACK_FAILED = 'ROLLBACK_FAILED'
     ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
     SNAPSHOT_COMPLETE = 'SNAPSHOT_COMPLETE'
     CHECK_COMPLETE = 'CHECK_COMPLETE'
@@ -35,10 +37,12 @@ class BayStatus(fields.Enum):
     ALL = (CREATE_IN_PROGRESS, CREATE_FAILED, CREATE_COMPLETE,
            UPDATE_IN_PROGRESS, UPDATE_FAILED, UPDATE_COMPLETE,
            DELETE_IN_PROGRESS, DELETE_FAILED, DELETE_COMPLETE,
-           RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_COMPLETE,
+           RESUME_COMPLETE, RESTORE_COMPLETE, ROLLBACK_IN_PROGRESS,
-           SNAPSHOT_COMPLETE, CHECK_COMPLETE, ADOPT_COMPLETE)
+           ROLLBACK_FAILED, ROLLBACK_COMPLETE, SNAPSHOT_COMPLETE,
+           CHECK_COMPLETE, ADOPT_COMPLETE)
 
-    STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED, DELETE_FAILED)
+    STATUS_FAILED = (CREATE_FAILED, UPDATE_FAILED,
+                     DELETE_FAILED, ROLLBACK_FAILED)
 
     def __init__(self):
         super(BayStatus, self).__init__(valid_values=BayStatus.ALL)

magnum/service/periodic.py

@@ -75,7 +75,8 @@ class MagnumPeriodicTasks(periodic_task.PeriodicTasks):
             osc = clients.OpenStackClients(ctx)
             status = [bay_status.CREATE_IN_PROGRESS,
                       bay_status.UPDATE_IN_PROGRESS,
-                      bay_status.DELETE_IN_PROGRESS]
+                      bay_status.DELETE_IN_PROGRESS,
+                      bay_status.ROLLBACK_IN_PROGRESS]
             filters = {'status': status}
             bays = objects.Bay.list(ctx, filters=filters)
             if not bays:

magnum/tests/unit/api/controllers/test_root.py

@@ -40,7 +40,7 @@ class TestRootController(api_base.FunctionalTest):
                               [{u'href': u'http://localhost/v1/',
                                 u'rel': u'self'}],
                            u'status': u'CURRENT',
-                           u'max_version': u'1.2',
+                           u'max_version': u'1.3',
                            u'min_version': u'1.1'}]}
 
         self.v1_expected = {

magnum/tests/unit/api/controllers/v1/test_bay.py

@@ -216,7 +216,7 @@ class TestPatch(api_base.FunctionalTest):
         self.mock_bay_update.side_effect = self._simulate_rpc_bay_update
         self.addCleanup(p.stop)
 
-    def _simulate_rpc_bay_update(self, bay):
+    def _simulate_rpc_bay_update(self, bay, rollback=False):
         bay.save()
         return bay
 
@@ -355,6 +355,17 @@ class TestPatch(api_base.FunctionalTest):
         self.assertEqual(400, response.status_int)
         self.assertTrue(response.json['errors'])
 
+    @mock.patch.object(rpcapi.API, 'bay_update_async')
+    def test_update_bay_with_rollback_enabled(self, mock_update):
+        response = self.patch_json(
+            '/bays/%s/?rollback=True' % self.bay.name,
+            [{'path': '/node_count', 'value': 4,
+              'op': 'replace'}],
+            headers={'OpenStack-API-Version': 'container-infra 1.3'})
+
+        mock_update.assert_called_once_with(mock.ANY, rollback=True)
+        self.assertEqual(202, response.status_code)
+
     def test_remove_ok(self):
         response = self.get_json('/bays/%s' % self.bay.uuid)
         self.assertIsNotNone(response['name'])

magnum/tests/unit/conductor/handlers/test_bay_conductor.py

@@ -76,7 +76,7 @@ class TestHandler(db_base.DbTestCase):
 
         mock_update_stack.assert_called_once_with(
             self.context, mock_openstack_client, self.bay,
-            mock_scale_manager.return_value)
+            mock_scale_manager.return_value, False)
         bay = objects.Bay.get(self.context, self.bay.uuid)
         self.assertEqual(2, bay.node_count)
 
@@ -142,7 +142,7 @@ class TestHandler(db_base.DbTestCase):
 
         mock_update_stack.assert_called_once_with(
             self.context, mock_openstack_client, self.bay,
-            mock_scale_manager.return_value)
+            mock_scale_manager.return_value, False)
         bay = objects.Bay.get(self.context, self.bay.uuid)
         self.assertEqual(2, bay.node_count)
 
@@ -604,6 +604,30 @@ class TestHeatPoller(base.TestCase):
         self.assertEqual(2, bay.node_count)
         self.assertEqual(1, poller.attempts)
 
+    def test_poll_done_by_rollback_complete(self):
+        mock_heat_stack, bay, poller = self.setup_poll_test()
+
+        mock_heat_stack.stack_status = bay_status.ROLLBACK_COMPLETE
+        mock_heat_stack.parameters = {'number_of_minions': 1}
+        self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
+
+        self.assertEqual(2, bay.save.call_count)
+        self.assertEqual(bay_status.ROLLBACK_COMPLETE, bay.status)
+        self.assertEqual(1, bay.node_count)
+        self.assertEqual(1, poller.attempts)
+
+    def test_poll_done_by_rollback_failed(self):
+        mock_heat_stack, bay, poller = self.setup_poll_test()
+
+        mock_heat_stack.stack_status = bay_status.ROLLBACK_FAILED
+        mock_heat_stack.parameters = {'number_of_minions': 1}
+        self.assertRaises(loopingcall.LoopingCallDone, poller.poll_and_check)
+
+        self.assertEqual(2, bay.save.call_count)
+        self.assertEqual(bay_status.ROLLBACK_FAILED, bay.status)
+        self.assertEqual(1, bay.node_count)
+        self.assertEqual(1, poller.attempts)
+
     def test_poll_destroy(self):
         mock_heat_stack, bay, poller = self.setup_poll_test()
 

magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py

@@ -669,7 +669,8 @@ class TestBayConductorWithK8s(base.TestCase):
             'parameters': {},
             'template': expected_template_contents,
             'files': {},
-            'environment_files': []
+            'environment_files': [],
+            'disable_rollback': True
         }
         mock_heat_client.stacks.update.assert_called_once_with(mock_stack_id,
                                                                **expected_args)

magnum/tests/unit/objects/test_objects.py

@@ -362,7 +362,7 @@ class TestObject(test_base.TestCase, _TestObject):
 # For more information on object version testing, read
 # http://docs.openstack.org/developer/magnum/objects.html
 object_data = {
-    'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d',
+    'Bay': '1.6-2386f79585a6c24bc7960884a4d0ebce',
     'BayModel': '1.14-ae175b4aaba2c60df37cac63ef734853',
     'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2',
     'MyObj': '1.0-b43567e512438205e32f4e95ca616697',

magnum/tests/unit/service/test_periodic.py

@@ -57,11 +57,15 @@ class PeriodicTestCase(base.TestCase):
         trust_attrs.update({'id': 4, 'stack_id': '44',
                            'status': bay_status.CREATE_COMPLETE})
         bay4 = utils.get_test_bay(**trust_attrs)
+        trust_attrs.update({'id': 5, 'stack_id': '55',
+                           'status': bay_status.ROLLBACK_IN_PROGRESS})
+        bay5 = utils.get_test_bay(**trust_attrs)
 
         self.bay1 = objects.Bay(ctx, **bay1)
         self.bay2 = objects.Bay(ctx, **bay2)
         self.bay3 = objects.Bay(ctx, **bay3)
         self.bay4 = objects.Bay(ctx, **bay4)
+        self.bay5 = objects.Bay(ctx, **bay5)
 
     @mock.patch.object(objects.Bay, 'list')
     @mock.patch('magnum.common.clients.OpenStackClients')
@@ -74,8 +78,10 @@ class PeriodicTestCase(base.TestCase):
                             stack_status_reason='fake_reason_11')
         stack3 = fake_stack(id='33', stack_status=bay_status.UPDATE_COMPLETE,
                             stack_status_reason='fake_reason_33')
-        mock_heat_client.stacks.list.return_value = [stack1, stack3]
+        stack5 = fake_stack(id='55', stack_status=bay_status.ROLLBACK_COMPLETE,
-        get_stacks = {'11': stack1, '33': stack3}
+                            stack_status_reason='fake_reason_55')
+        mock_heat_client.stacks.list.return_value = [stack1, stack3, stack5]
+        get_stacks = {'11': stack1, '33': stack3, '55': stack5}
 
         def stack_get_sideefect(arg):
             if arg == '22':
@@ -85,7 +91,8 @@ class PeriodicTestCase(base.TestCase):
         mock_heat_client.stacks.get.side_effect = stack_get_sideefect
         mock_osc = mock_oscc.return_value
         mock_osc.heat.return_value = mock_heat_client
-        mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
+        mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
+                                      self.bay5]
 
         mock_keystone_client = mock.MagicMock()
         mock_keystone_client.client.project_id = "fake_project"
@@ -98,6 +105,8 @@ class PeriodicTestCase(base.TestCase):
         mock_db_destroy.assert_called_once_with(self.bay2.uuid)
         self.assertEqual(bay_status.UPDATE_COMPLETE, self.bay3.status)
         self.assertEqual('fake_reason_33', self.bay3.status_reason)
+        self.assertEqual(bay_status.ROLLBACK_COMPLETE, self.bay5.status)
+        self.assertEqual('fake_reason_55', self.bay5.status_reason)
 
     @mock.patch.object(objects.Bay, 'list')
     @mock.patch('magnum.common.clients.OpenStackClients')
@@ -136,7 +145,9 @@ class PeriodicTestCase(base.TestCase):
                             stack_status=bay_status.DELETE_IN_PROGRESS)
         stack3 = fake_stack(id='33',
                             stack_status=bay_status.UPDATE_IN_PROGRESS)
-        get_stacks = {'11': stack1, '22': stack2, '33': stack3}
+        stack5 = fake_stack(id='55',
+                            stack_status=bay_status.ROLLBACK_IN_PROGRESS)
+        get_stacks = {'11': stack1, '22': stack2, '33': stack3, '55': stack5}
 
         def stack_get_sideefect(arg):
             if arg == '22':
@@ -144,15 +155,18 @@ class PeriodicTestCase(base.TestCase):
             return get_stacks[arg]
 
         mock_heat_client.stacks.get.side_effect = stack_get_sideefect
-        mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3]
+        mock_heat_client.stacks.list.return_value = [stack1, stack2, stack3,
+                                                     stack5]
         mock_osc = mock_oscc.return_value
         mock_osc.heat.return_value = mock_heat_client
-        mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3]
+        mock_bay_list.return_value = [self.bay1, self.bay2, self.bay3,
+                                      self.bay5]
         periodic.MagnumPeriodicTasks(CONF).sync_bay_status(None)
 
         self.assertEqual(bay_status.CREATE_IN_PROGRESS, self.bay1.status)
         self.assertEqual(bay_status.DELETE_IN_PROGRESS, self.bay2.status)
         self.assertEqual(bay_status.UPDATE_IN_PROGRESS, self.bay3.status)
+        self.assertEqual(bay_status.ROLLBACK_IN_PROGRESS, self.bay5.status)
 
     @mock.patch.object(objects.Bay, 'list')
     @mock.patch('magnum.common.clients.OpenStackClients')

releasenotes/notes/rollback-bay-on-update-failure-83e5ff8a7904d5c4.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - Add Microversion 1.3 to support Magnum bay rollback,
+    user can enable rollback on bay update failure by
+    setting 'OpenStack-API-Version' to 'container-infra 1.3'
+    in request header and passing 'rollback=True' param
+    in bay update request.