From b706155888d740841d745c52aae543cf82fab0bc Mon Sep 17 00:00:00 2001 From: John Garbutt Date: Tue, 7 Feb 2017 18:55:26 +0000 Subject: [PATCH] Stop _undefine_domain erroring if domain not found During live-migration stress testing we are seeing the following log: Error from libvirt during undefine. Code=42 Error=Domain not found There appears to be a race while trying to undefine the domain, and something else is possibly also doing some kind of clean up. While this does paper over that race, it stops otherwise completed live-migrations from failing. It also matches similar error handling done for when deleting the domain. The next part of the bug fix is to ensure if we have any similar unexpected errors during this later phase of the live-migration we don't leave the instance stuck in the migrating state, it should move to an ERROR state. This is covered in a follow on patch. Partial-Bug: #1662626 Change-Id: I23ed9819061bfa436b12180110666c5b8c3e0f70 --- nova/tests/unit/virt/libvirt/test_driver.py | 24 +++++++++++++++++++++ nova/virt/libvirt/driver.py | 14 ++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/nova/tests/unit/virt/libvirt/test_driver.py b/nova/tests/unit/virt/libvirt/test_driver.py index fdcb872a83cd..42092c7fc3b1 100644 --- a/nova/tests/unit/virt/libvirt/test_driver.py +++ b/nova/tests/unit/virt/libvirt/test_driver.py @@ -12247,6 +12247,30 @@ class LibvirtConnTestCase(test.NoDBTestCase): # instance disappears drvr._undefine_domain(instance) + @mock.patch.object(libvirt_driver.LibvirtDriver, "_has_uefi_support") + @mock.patch.object(host.Host, "get_guest") + def test_undefine_domain_handles_libvirt_errors(self, mock_get, + mock_has_uefi): + drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False) + instance = objects.Instance(**self.test_instance) + fake_guest = mock.Mock() + mock_get.return_value = fake_guest + + unexpected = fakelibvirt.make_libvirtError( + fakelibvirt.libvirtError, "Random", error_code=1) + fake_guest.delete_configuration.side_effect = unexpected + + # ensure raise unexpected error code + self.assertRaises(type(unexpected), drvr._undefine_domain, instance) + + ignored = fakelibvirt.make_libvirtError( + fakelibvirt.libvirtError, "No such domain", + error_code=fakelibvirt.VIR_ERR_NO_DOMAIN) + fake_guest.delete_configuration.side_effect = ignored + + # ensure no raise for no such domain + drvr._undefine_domain(instance) + @mock.patch.object(host.Host, "list_instance_domains") @mock.patch.object(objects.BlockDeviceMappingList, "bdms_by_instance_uuid") @mock.patch.object(objects.InstanceList, "get_by_filters") diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py index 0aad4105a436..9c744fba1b98 100644 --- a/nova/virt/libvirt/driver.py +++ b/nova/virt/libvirt/driver.py @@ -915,11 +915,17 @@ class LibvirtDriver(driver.ComputeDriver): support_uefi = self._has_uefi_support() guest.delete_configuration(support_uefi) except libvirt.libvirtError as e: - with excutils.save_and_reraise_exception(): + with excutils.save_and_reraise_exception() as ctxt: errcode = e.get_error_code() - LOG.error(_LE('Error from libvirt during undefine. ' - 'Code=%(errcode)s Error=%(e)s'), - {'errcode': errcode, 'e': e}, instance=instance) + if errcode == libvirt.VIR_ERR_NO_DOMAIN: + LOG.debug("Called undefine, but domain already gone.", + instance=instance) + ctxt.reraise = False + else: + LOG.error(_LE('Error from libvirt during undefine. ' + 'Code=%(errcode)s Error=%(e)s'), + {'errcode': errcode, 'e': e}, + instance=instance) except exception.InstanceNotFound: pass