Browse Source

Stop _undefine_domain erroring if domain not found

During live-migration stress testing we are seeing the following log:
Error from libvirt during undefine. Code=42 Error=Domain not found

There appears to be a race while trying to undefine the domain, and
something else is possibly also doing some kind of clean up. While this
does paper over that race, it stops otherwise completed live-migrations
from failing. It also matches similar error handling done for when
deleting the domain.

The next part of the bug fix is to ensure if we have any similar
unexpected errors during this later phase of the live-migration we don't
leave the instance stuck in the migrating state, it should move to an
ERROR state. This is covered in a follow on patch.

Partial-Bug: #1662626

Change-Id: I23ed9819061bfa436b12180110666c5b8c3e0f70
(cherry picked from commit b706155888)
changes/40/508640/6
John Garbutt 4 years ago
committed by Arnaud Morin
parent
commit
a0525f650a
2 changed files with 34 additions and 4 deletions
  1. +24
    -0
      nova/tests/unit/virt/libvirt/test_driver.py
  2. +10
    -4
      nova/virt/libvirt/driver.py

+ 24
- 0
nova/tests/unit/virt/libvirt/test_driver.py View File

@ -12355,6 +12355,30 @@ class LibvirtConnTestCase(test.NoDBTestCase):
# instance disappears
drvr._undefine_domain(instance)
@mock.patch.object(libvirt_driver.LibvirtDriver, "_has_uefi_support")
@mock.patch.object(host.Host, "get_guest")
def test_undefine_domain_handles_libvirt_errors(self, mock_get,
mock_has_uefi):
drvr = libvirt_driver.LibvirtDriver(fake.FakeVirtAPI(), False)
instance = objects.Instance(**self.test_instance)
fake_guest = mock.Mock()
mock_get.return_value = fake_guest
unexpected = fakelibvirt.make_libvirtError(
fakelibvirt.libvirtError, "Random", error_code=1)
fake_guest.delete_configuration.side_effect = unexpected
# ensure raise unexpected error code
self.assertRaises(type(unexpected), drvr._undefine_domain, instance)
ignored = fakelibvirt.make_libvirtError(
fakelibvirt.libvirtError, "No such domain",
error_code=fakelibvirt.VIR_ERR_NO_DOMAIN)
fake_guest.delete_configuration.side_effect = ignored
# ensure no raise for no such domain
drvr._undefine_domain(instance)
@mock.patch.object(host.Host, "list_instance_domains")
@mock.patch.object(objects.BlockDeviceMappingList, "bdms_by_instance_uuid")
@mock.patch.object(objects.InstanceList, "get_by_filters")


+ 10
- 4
nova/virt/libvirt/driver.py View File

@ -915,11 +915,17 @@ class LibvirtDriver(driver.ComputeDriver):
support_uefi = self._has_uefi_support()
guest.delete_configuration(support_uefi)
except libvirt.libvirtError as e:
with excutils.save_and_reraise_exception():
with excutils.save_and_reraise_exception() as ctxt:
errcode = e.get_error_code()
LOG.error(_LE('Error from libvirt during undefine. '
'Code=%(errcode)s Error=%(e)s'),
{'errcode': errcode, 'e': e}, instance=instance)
if errcode == libvirt.VIR_ERR_NO_DOMAIN:
LOG.debug("Called undefine, but domain already gone.",
instance=instance)
ctxt.reraise = False
else:
LOG.error(_LE('Error from libvirt during undefine. '
'Code=%(errcode)s Error=%(e)s'),
{'errcode': errcode, 'e': e},
instance=instance)
except exception.InstanceNotFound:
pass


Loading…
Cancel
Save