Handle compute node not available for live migration

This patch handles exception.ComputeServiceUnavailable by restoring
instance's vm_state and instance's task_state after live migration
failure caused by unavailable source/dest compute node.

Raises detailed HTTPBadRequest explanation for this exception.

Fixes bug 973393 and bug 1051881

Change-Id: If825b61fad9c4e3030f2e6c5002907255eaf3661

nova/exception.py

@@ -358,7 +358,7 @@ class ComputeResourcesUnavailable(ServiceUnavailable):
 
 
 class ComputeServiceUnavailable(ServiceUnavailable):
-    message = _("Compute service is unavailable at this time.")
+    message = _("Compute service of %(host)s is unavailable at this time.")
 
 
 class UnableToMigrateToSelf(Invalid):

nova/scheduler/driver.py

@@ -209,7 +209,10 @@ class Scheduler(object):
         """
 
         # Checking dest exists and compute node.
-        dservice_ref = db.service_get_by_compute_host(context, dest)
+        try:
+            dservice_ref = db.service_get_by_compute_host(context, dest)
+        except exception.NotFound:
+            raise exception.ComputeServiceUnavailable(host=dest)
 
         # Checking dest host is alive.
         if not self.servicegroup_api.service_is_up(dservice_ref):

nova/scheduler/manager.py

@@ -24,6 +24,7 @@ Scheduler Service
 import sys
 
 from nova.compute import rpcapi as compute_rpcapi
+from nova.compute import task_states
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova.conductor import api as conductor_api
@@ -92,6 +93,16 @@ class SchedulerManager(manager.Manager):
             return self.driver.schedule_live_migration(
                 context, instance, dest,
                 block_migration, disk_over_commit)
+        except exception.ComputeServiceUnavailable as ex:
+            request_spec = {'instance_properties': {
+                'uuid': instance['uuid'], },
+            }
+            with excutils.save_and_reraise_exception():
+                self._set_vm_state_and_notify('live_migration',
+                            dict(vm_state=instance['vm_state'],
+                                 task_state=None,
+                                 expected_task_state=task_states.MIGRATING,),
+                                              context, ex, request_spec)
         except Exception as ex:
             with excutils.save_and_reraise_exception():
                 self._set_vm_state_and_notify('live_migration',

nova/tests/scheduler/test_scheduler.py

@@ -24,6 +24,7 @@ import mox
 from nova.compute import api as compute_api
 from nova.compute import power_state
 from nova.compute import rpcapi as compute_rpcapi
+from nova.compute import task_states
 from nova.compute import utils as compute_utils
 from nova.compute import vm_states
 from nova.conductor import api as conductor_api
@@ -197,6 +198,38 @@ class SchedulerManagerTestCase(test.TestCase):
         self.manager.run_instance(self.context, request_spec,
                 None, None, None, None, {})
 
+    def test_live_migration_compute_service_notavailable(self):
+        inst = {"uuid": "fake-instance-id",
+                "vm_state": vm_states.ACTIVE,
+                "task_state": task_states.MIGRATING, }
+
+        dest = 'fake_host'
+        block_migration = False
+        disk_over_commit = False
+
+        self._mox_schedule_method_helper('schedule_live_migration')
+        self.mox.StubOutWithMock(compute_utils, 'add_instance_fault_from_exc')
+        self.mox.StubOutWithMock(db, 'instance_update_and_get_original')
+
+        self.manager.driver.schedule_live_migration(self.context,
+                    inst, dest, block_migration, disk_over_commit).AndRaise(
+                    exception.ComputeServiceUnavailable(host="src"))
+        db.instance_update_and_get_original(self.context, inst["uuid"],
+                                {"vm_state": inst['vm_state'],
+                                 "task_state": None,
+                                 "expected_task_state": task_states.MIGRATING,
+                                }).AndReturn((inst, inst))
+        compute_utils.add_instance_fault_from_exc(self.context,
+                                mox.IsA(conductor_api.LocalAPI), inst,
+                                mox.IsA(exception.ComputeServiceUnavailable),
+                                mox.IgnoreArg())
+
+        self.mox.ReplayAll()
+        self.assertRaises(exception.ComputeServiceUnavailable,
+                          self.manager.live_migration,
+                          self.context, inst, dest, block_migration,
+                          disk_over_commit)
+
     def test_prep_resize_no_valid_host_back_in_active_state(self):
         fake_instance_uuid = 'fake-instance-id'
         inst = {"vm_state": "", "task_state": ""}
@@ -502,6 +535,29 @@ class SchedulerTestCase(test.TestCase):
                 block_migration=block_migration,
                 disk_over_commit=disk_over_commit)
 
+    def test_live_migration_compute_dest_not_exist(self):
+        # Raise exception when dest compute node does not exist.
+
+        self.mox.StubOutWithMock(self.driver, '_live_migration_src_check')
+        self.mox.StubOutWithMock(db, 'service_get_by_compute_host')
+
+        dest = 'fake_host2'
+        block_migration = False
+        disk_over_commit = False
+        instance = self._live_migration_instance()
+
+        self.driver._live_migration_src_check(self.context, instance)
+        # Compute down
+        db.service_get_by_compute_host(self.context,
+                            dest).AndRaise(exception.NotFound())
+
+        self.mox.ReplayAll()
+        self.assertRaises(exception.ComputeServiceUnavailable,
+                self.driver.schedule_live_migration, self.context,
+                instance=instance, dest=dest,
+                block_migration=block_migration,
+                disk_over_commit=disk_over_commit)
+
     def test_live_migration_compute_dest_not_alive(self):
         # Raise exception when dest compute node is not alive.