Cells: check states on resize/rebuild updates

When the API cell receives instance updates, check the task states for
any resize and rebuild states.  Make sure to only apply them if the
current task state for the instance is what we expect.  This will fix
any out of order messages from potentially stomping on re-setting the
instance to ACTIVE/None on finish of the actions.

Change-Id: Id79e35e29170ac2b8f42a955cecaae3072aba09c
Closes-bug: 1249072

nova/cells/messaging.py

@@ -936,6 +936,56 @@ class _BroadcastMessageMethods(_BaseMessageMethods):
         """Are we the API level?"""
         return not self.state_manager.get_parent_cells()
 
+    def _apply_expected_states(self, instance_info):
+        """To attempt to address out-of-order messages, do some sanity
+        checking on the VM and task states.  Add some requirements for
+        vm_state and task_state to the instance_update() DB call if
+        necessary.
+        """
+        expected_vm_state_map = {
+                # For updates containing 'vm_state' of 'building',
+                # only allow them to occur if the DB already says
+                # 'building' or if the vm_state is None.  None
+                # really shouldn't be possible as instances always
+                # start out in 'building' anyway.. but just in case.
+                vm_states.BUILDING: [vm_states.BUILDING, None]}
+
+        expected_task_state_map = {
+                # Always allow updates when task_state doesn't change,
+                # but also make sure we don't set resize/rebuild task
+                # states for old messages when we've potentially already
+                # processed the ACTIVE/None messages.  Ie, these checks
+                # will prevent stomping on any ACTIVE/None messages
+                # we already processed.
+                task_states.REBUILD_BLOCK_DEVICE_MAPPING:
+                        [task_states.REBUILD_BLOCK_DEVICE_MAPPING,
+                         task_states.REBUILDING],
+                task_states.REBUILD_SPAWNING:
+                        [task_states.REBUILD_SPAWNING,
+                         task_states.REBUILD_BLOCK_DEVICE_MAPPING,
+                         task_states.REBUILDING],
+                task_states.RESIZE_MIGRATING:
+                        [task_states.RESIZE_MIGRATING,
+                         task_states.RESIZE_PREP],
+                task_states.RESIZE_MIGRATED:
+                        [task_states.RESIZE_MIGRATED,
+                         task_states.RESIZE_MIGRATING,
+                         task_states.RESIZE_PREP],
+                task_states.RESIZE_FINISH:
+                        [task_states.RESIZE_FINISH,
+                         task_states.RESIZE_MIGRATED,
+                         task_states.RESIZE_MIGRATING,
+                         task_states.RESIZE_PREP]}
+
+        if 'vm_state' in instance_info:
+            expected = expected_vm_state_map.get(instance_info['vm_state'])
+            if expected is not None:
+                instance_info['expected_vm_state'] = expected
+        if 'task_state' in instance_info:
+            expected = expected_task_state_map.get(instance_info['task_state'])
+            if expected is not None:
+                instance_info['expected_task_state'] = expected
+
     def instance_update_at_top(self, message, instance, **kwargs):
         """Update an instance in the DB if we're a top level cell."""
         if not self._at_the_top():
@@ -967,20 +1017,7 @@ class _BroadcastMessageMethods(_BaseMessageMethods):
         LOG.debug(_("Got update for instance: %(instance)s"),
                   {'instance': instance}, instance_uuid=instance_uuid)
 
-        # To attempt to address out-of-order messages, do some sanity
-        # checking on the VM state.
-        expected_vm_state_map = {
-                # For updates containing 'vm_state' of 'building',
-                # only allow them to occur if the DB already says
-                # 'building' or if the vm_state is None.  None
-                # really shouldn't be possible as instances always
-                # start out in 'building' anyway.. but just in case.
-                vm_states.BUILDING: [vm_states.BUILDING, None]}
-
-        expected_vm_states = expected_vm_state_map.get(
-                instance.get('vm_state'))
-        if expected_vm_states:
-                instance['expected_vm_state'] = expected_vm_states
+        self._apply_expected_states(instance)
 
         # It's possible due to some weird condition that the instance
         # was already set as deleted... so we'll attempt to update

nova/tests/cells/test_cells_messaging.py

@@ -1398,6 +1398,23 @@ class CellsBroadcastMethodsTestCase(test.TestCase):
         self.assertFalse(self.mid_methods_cls._at_the_top())
         self.assertFalse(self.src_methods_cls._at_the_top())
 
+    def test_apply_expected_states_building(self):
+        instance_info = {'vm_state': vm_states.BUILDING}
+        expected = dict(instance_info,
+                        expected_vm_state=[vm_states.BUILDING, None])
+        self.src_methods_cls._apply_expected_states(instance_info)
+        self.assertEqual(expected, instance_info)
+
+    def test_apply_expected_states_resize_finish(self):
+        instance_info = {'task_state': task_states.RESIZE_FINISH}
+        exp_states = [task_states.RESIZE_FINISH,
+                      task_states.RESIZE_MIGRATED,
+                      task_states.RESIZE_MIGRATING,
+                      task_states.RESIZE_PREP]
+        expected = dict(instance_info, expected_task_state=exp_states)
+        self.src_methods_cls._apply_expected_states(instance_info)
+        self.assertEqual(expected, instance_info)
+
     def _test_instance_update_at_top(self, net_info, exists=True):
         fake_info_cache = {'id': 1,
                            'instance': 'fake_instance',