Fix run/terminate race conditions.

* synchronize run,terminate,stop,start on instance_uuid
 * don't surpress error when unfiltering instance, which
   can result in a zombified instance.
 * Fixes bug 956719
 * Remove debug raise

Change-Id: I8b2eaffdabfd5c1a9414adb1b5ed11e4c48711fc

nova/exception.py

@@ -330,6 +330,10 @@ class InstanceRebootFailure(Invalid):
     message = _("Failed to reboot instance") + ": %(reason)s"
 
 
+class InstanceTerminationFailure(Invalid):
+    message = _("Failed to terminate instance") + ": %(reason)s"
+
+
 class ServiceUnavailable(Invalid):
     message = _("Service is unavailable at this time.")
 

nova/tests/test_compute.py

@@ -932,26 +932,21 @@ class ComputeTestCase(BaseTestCase):
 
         self.compute.terminate_instance(self.context, instance['uuid'])
 
-    def test_instance_set_to_error_on_deleted_instance_doesnt_raise(self):
-        """Test that we don't raise InstanceNotFound when trying to set
-        an instance to ERROR that has already been deleted from under us.
-        The original exception should be re-raised.
+    def test_instance_termination_exception_sets_error(self):
+        """Test that we handle InstanceTerminationFailure
+        which is propagated up from the underlying driver.
         """
         instance = self._create_fake_instance()
-        instance_uuid = instance['uuid']
 
-        def fake_allocate_network(context, instance, requested_networks):
-            # Remove the instance to simulate race condition
+        def fake_delete_instance(context, instance):
+            raise exception.InstanceTerminationFailure(reason='')
+
+        self.stubs.Set(self.compute, '_delete_instance',
+                       fake_delete_instance)
+
         self.compute.terminate_instance(self.context, instance['uuid'])
-            raise rpc_common.RemoteError()
-
-        self.stubs.Set(self.compute, '_allocate_network',
-                       fake_allocate_network)
-
-        self.assertRaises(rpc_common.RemoteError,
-                          self.compute.run_instance,
-                          self.context,
-                          instance_uuid)
+        instance = db.instance_get_by_uuid(self.context, instance['uuid'])
+        self.assertEqual(instance['vm_state'], vm_states.ERROR)
 
     def test_network_is_deallocated_on_spawn_failure(self):
         """When a spawn fails the network must be deallocated"""