openstack
/
nova
Code Issues Proposed changes

vmware: Handle exception in destroy with attached volumes 

When an attempt to delete an instance doesn't succeed and nova retries
on the next nova-compute restart, an instance not existing in the back
end anymore can lead to an uncatched exception in the vmware driver
prohibiting instance deletion. This is the case, if the instance had
volumes attached, because `_detach_instance_volumes()` always powers off
the instance - which cannot work if the instance doesn't exist anymore.

While the code already catched `ManagedObjectNotFoundException`, it also
needs to catch `InstanceNotFound` raised by `vm_util.get_vm_ref()` to
complete the deletion as seen in the traceback below (which comes from a
queens codebase):

     Traceback (most recent call last):
       File "/nova/compute/manager.py", line 874, in _init_instance
         self._delete_instance(context, instance, bdms)
       File "/nova/hooks.py", line 154, in inner
         rv = f(*args, **kwargs)
       File "/nova/compute/manager.py", line 2500, in _delete_instance
         self._shutdown_instance(context, instance, bdms)
       File "/nova/compute/manager.py", line 2392, in _shutdown_instance
         requested_networks)
       File "/lib/python2.7/site-packages/oslo_utils/excutils.py", line 220, in __exit__
         self.force_reraise()
       File "/lib/python2.7/site-packages/oslo_utils/excutils.py", line 196, in force_reraise
         six.reraise(self.type_, self.value, self.tb)
       File "/nova/compute/manager.py", line 2379, in _shutdown_instance
         block_device_info)
       File "/nova/virt/vmwareapi/driver.py", line 574, in destroy
         self._detach_instance_volumes(instance, block_device_info)
       File "/nova/virt/vmwareapi/driver.py", line 536, in _detach_instance_volumes
         self._vmops.power_off(instance)
       File "/nova/virt/vmwareapi/vmops.py", line 1762, in power_off
         vm_util.power_off_instance(self._session, instance)
       File "/nova/virt/vmwareapi/vm_util.py", line 1732, in power_off_instance
         vm_ref = get_vm_ref(session, instance)
       File "/nova/virt/vmwareapi/vm_util.py", line 171, in wrapper
         return _vm_ref_cache(id, func, session, instance)
       File "/nova/virt/vmwareapi/vm_util.py", line 162, in _vm_ref_cache
         vm_ref = func(session, data)
       File "/nova/virt/vmwareapi/vm_util.py", line 1214, in get_vm_ref
         raise exception.InstanceNotFound(instance_id=uuid)
     InstanceNotFound: Instance 2af34cc5-22e0-400c-8b80-f130e86027fd could not be found.

Change-Id: I65d2f76068e4b033ffd20959c9e74c870c8aa8e0
This commit is contained in:
Johannes Kulik 2020-06-29 12:42:35 +02:00
parent 1490a54fa9
commit a323ed29ee
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
nova/tests/unit/virt/vmwareapi/test_driver_api.py
View File

@ -1593,6 +1593,17 @@ class VMwareAPIVMTestCase(test.NoDBTestCase,
self._destroy_instance_without_vm_ref(
task_state=task_states.RESIZE_REVERTING)
def test_destroy_instance_with_vm_ref_and_with_volumes(self):
self.destroy_disks = True
self._create_instance()
bdi = {'block_device_mapping': ['foo']}
with mock.patch.object(self.conn._vmops,
"destroy") as mock_destroy:
self.conn.destroy(self.context, self.instance, self.network_info,
bdi, self.destroy_disks)
mock_destroy.assert_called_once_with(self.instance,
self.destroy_disks)
def _rescue(self, config_drive=False):
# validate that the power on is only called once
self._power_on = vm_util.power_on_instance

3
nova/virt/vmwareapi/driver.py
View File

@ -623,7 +623,8 @@ class VMwareVCDriver(driver.ComputeDriver):
if block_device_info is not None:
try:
self._detach_instance_volumes(instance, block_device_info)
except vexc.ManagedObjectNotFoundException:
except (vexc.ManagedObjectNotFoundException,
exception.InstanceNotFound):
LOG.warning('Instance does not exists. Proceeding to '
'delete instance properties on datastore',
instance=instance)