Browse Source

Stop _undefine_domain erroring if domain not found

During live-migration stress testing we are seeing the following log:
Error from libvirt during undefine. Code=42 Error=Domain not found

There appears to be a race while trying to undefine the domain, and
something else is possibly also doing some kind of clean up. While this
does paper over that race, it stops otherwise completed live-migrations
from failing. It also matches similar error handling done for when
deleting the domain.

The next part of the bug fix is to ensure if we have any similar
unexpected errors during this later phase of the live-migration we don't
leave the instance stuck in the migrating state, it should move to an
ERROR state. This is covered in a follow on patch.

Partial-Bug: #1662626

Change-Id: I23ed9819061bfa436b12180110666c5b8c3e0f70
changes/00/430400/3
John Garbutt 4 years ago
committed by John Garbutt
parent
commit
b706155888
2 changed files with 34 additions and 4 deletions
  1. +24
    -0
      nova/tests/unit/virt/libvirt/test_driver.py
  2. +10
    -4
      nova/virt/libvirt/driver.py

+ 24
- 0
nova/tests/unit/virt/libvirt/test_driver.py View File

@ -12247,6 +12247,30 @@ class LibvirtConnTestCase(test.NoDBTestCase):
# instance disappears
drvr._undefine_domain(instance)
@mock.patch.object(libvirt_driver.LibvirtDriver, "_has_uefi_support")
@mock.patch.object(host.Host, "get_guest")
def test_undefine_domain_handles_libvirt_errors(self, mock_get,
mock_has_uefi):
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
instance = objects.Instance(**self.test_instance)
fake_guest = mock.Mock()
mock_get.return_value = fake_guest
unexpected = fakelibvirt.make_libvirtError(
fakelibvirt.libvirtError, "Random", error_code=1)
fake_guest.delete_configuration.side_effect = unexpected
# ensure raise unexpected error code
self.assertRaises(type(unexpected), drvr._undefine_domain, instance)
ignored = fakelibvirt.make_libvirtError(
fakelibvirt.libvirtError, "No such domain",
error_code=fakelibvirt.VIR_ERR_NO_DOMAIN)
fake_guest.delete_configuration.side_effect = ignored
# ensure no raise for no such domain
drvr._undefine_domain(instance)
@mock.patch.object(host.Host, "list_instance_domains")
@mock.patch.object(objects.BlockDeviceMappingList, "bdms_by_instance_uuid")
@mock.patch.object(objects.InstanceList, "get_by_filters")


+ 10
- 4
nova/virt/libvirt/driver.py View File

@ -915,11 +915,17 @@ class LibvirtDriver(driver.ComputeDriver):
support_uefi = self._has_uefi_support()
guest.delete_configuration(support_uefi)
except libvirt.libvirtError as e:
with excutils.save_and_reraise_exception():
with excutils.save_and_reraise_exception() as ctxt:
errcode = e.get_error_code()
LOG.error(_LE('Error from libvirt during undefine. '
'Code=%(errcode)s Error=%(e)s'),
{'errcode': errcode, 'e': e}, instance=instance)
if errcode == libvirt.VIR_ERR_NO_DOMAIN:
LOG.debug("Called undefine, but domain already gone.",
instance=instance)
ctxt.reraise = False
else:
LOG.error(_LE('Error from libvirt during undefine. '
'Code=%(errcode)s Error=%(e)s'),
{'errcode': errcode, 'e': e},
instance=instance)
except exception.InstanceNotFound:
pass


Loading…
Cancel
Save