libvirt: Ignore volume exceptions during post_live_migration

Previously errors while disconnecting volumes from the source host
during post_live_migration within LibvirtDriver would result in the
overall failure of the migration. This would also mean that while the
instance would be running on the destination it would still be listed as
running on the source within the db.

This change simply ignores any exceptions raised while attempting to
disconnect volumes on the source. These errors can be safely ignored as
they will have no impact on the running instance on the destination.

In the future Nova could wire up the force and ignore_errors kwargs when
calling down into the associated os-brick connectors to help avoid this.

Closes-Bug: #1843639
Change-Id: Ieff5243854321ec40f642845e87a0faecaca8721

nova/tests/unit/virt/libvirt/test_driver.py

@@ -13631,6 +13631,33 @@ class LibvirtConnTestCase(test.NoDBTestCase,
 
         _test()
 
+    @mock.patch.object(libvirt_driver.LibvirtDriver, '_disconnect_volume')
+    @mock.patch.object(driver, 'block_device_info_get_mapping')
+    def test_post_live_migration_exception_swallowed(self, mock_get_bdm,
+                                                     mock_disconnect_volume):
+        vol_1_conn_info = {'data': {'volume_id': uuids.vol_1_id}}
+        vol_2_conn_info = {'data': {'volume_id': uuids.vol_2_id}}
+        mock_get_bdm.return_value = [{'connection_info': vol_1_conn_info},
+                                     {'connection_info': vol_2_conn_info}]
+
+        # Raise an exception with the first call to disconnect_volume
+        mock_disconnect_volume.side_effect = [test.TestingException, None]
+
+        drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
+        drvr.post_live_migration(mock.sentinel.ctxt, mock.sentinel.instance,
+                                 mock.sentinel.bdi)
+
+        # Assert disconnect_volume is called twice despite the exception
+        mock_disconnect_volume.assert_has_calls([
+            mock.call(mock.sentinel.ctxt, vol_1_conn_info,
+                      mock.sentinel.instance),
+            mock.call(mock.sentinel.ctxt, vol_2_conn_info,
+                      mock.sentinel.instance)])
+
+        # Assert that we log the failure to disconnect the first volume
+        self.assertIn("Ignoring exception while attempting to disconnect "
+                      "volume %s" % uuids.vol_1_id, self.stdlog.logger.output)
+
     @mock.patch('os.stat')
     @mock.patch('os.path.getsize')
     @mock.patch('nova.virt.disk.api.get_disk_info')

nova/virt/libvirt/driver.py

@@ -9335,15 +9335,25 @@ class LibvirtDriver(driver.ComputeDriver):
 
     def post_live_migration(self, context, instance, block_device_info,
                             migrate_data=None):
-        # Disconnect from volume server
+        # NOTE(mdbooth): The block_device_info we were passed was initialized
+        # with BDMs from the source host before they were updated to point to
+        # the destination. We can safely use this to disconnect the source
+        # without re-fetching.
         block_device_mapping = driver.block_device_info_get_mapping(
                 block_device_info)
+
         for vol in block_device_mapping:
-            # NOTE(mdbooth): The block_device_info we were passed was
-            # initialized with BDMs from the source host before they were
-            # updated to point to the destination. We can safely use this to
-            # disconnect the source without re-fetching.
-            self._disconnect_volume(context, vol['connection_info'], instance)
+            connection_info = vol['connection_info']
+            # NOTE(lyarwood): Ignore exceptions here to avoid the instance
+            # being left in an ERROR state and still marked on the source.
+            try:
+                self._disconnect_volume(context, connection_info, instance)
+            except Exception as ex:
+                volume_id = driver_block_device.get_volume_id(connection_info)
+                LOG.exception("Ignoring exception while attempting to "
+                              "disconnect volume %s from the source host "
+                              "during post_live_migration", volume_id,
+                              instance=instance)
 
     def post_live_migration_at_source(self, context, instance, network_info):
         """Unplug VIFs from networks at source.