Disconnecting volume from the compute host

cmd nova-manage volume_attachment refresh vm-id vol-id connetor

There were cases where the instance said to live in compute#1 but the
connection_info in the BDM record was for compute#2, and when the script
called `remote_volume_connection` then nova would call os-brick on
compute#1 (the wrong node) and try to detach it.

In some case os-brick would mistakenly think that the volume was
attached (because the target and lun matched an existing volume on the
host) and would try to disconnect, resulting in errors on the compute
logs.

- Added HostConflict exception
- Fixes dedent in cmd/manange.py
- Updates nova-mange doc

Closes-Bug: #2012365
Change-Id: I21109752ff1c56d3cefa58fcd36c68bf468e0a73
This commit is contained in:
Amit Uniyal 2023-03-15 05:07:28 +00:00
parent d29a9b64ee
commit a8f81d5f08
4 changed files with 70 additions and 10 deletions

View File

@ -1572,7 +1572,9 @@ command.
* - 5
- Instance state invalid (must be stopped and unlocked)
* - 6
- Instance is not attached to volume
- Volume is not attached to the instance
* - 7
- Connector host is not correct
Libvirt Commands

View File

@ -164,18 +164,14 @@ def locked_instance(cell_mapping, instance, reason):
initial_state = 'locked' if instance.locked else 'unlocked'
if not instance.locked:
with context.target_cell(
context.get_admin_context(),
cell_mapping
) as cctxt:
context.get_admin_context(), cell_mapping) as cctxt:
compute_api.lock(cctxt, instance, reason=reason)
try:
yield
finally:
if initial_state == 'unlocked':
with context.target_cell(
context.get_admin_context(),
cell_mapping
) as cctxt:
context.get_admin_context(), cell_mapping) as cctxt:
compute_api.unlock(cctxt, instance)
@ -3139,8 +3135,15 @@ class VolumeAttachmentCommands(object):
# TODO(lyarwood): Add delete_attachment as a kwarg to
# remove_volume_connection as is available in the private
# method within the manager.
if instance.host == connector['host']:
compute_rpcapi.remove_volume_connection(
cctxt, instance, volume_id, instance.host)
else:
msg = (
f"The compute host '{connector['host']}' in the "
f"connector does not match the instance host "
f"'{instance.host}'.")
raise exception.HostConflict(_(msg))
# Delete the existing volume attachment if present in the bdm.
# This isn't present when the original attachment was made
@ -3222,6 +3225,7 @@ class VolumeAttachmentCommands(object):
* 4: Instance does not exist.
* 5: Instance state invalid.
* 6: Volume is not attached to instance.
* 7: Connector host is not correct.
"""
try:
# TODO(lyarwood): Make this optional and provide a rpcapi capable
@ -3237,6 +3241,12 @@ class VolumeAttachmentCommands(object):
# Refresh the volume attachment
return self._refresh(instance_uuid, volume_id, connector)
except exception.HostConflict as e:
print(
f"The command 'nova-manage volume_attachment get_connector' "
f"may have been run on the wrong compute host. Or the "
f"instance host may be wrong and in need of repair.\n{e}")
return 7
except exception.VolumeBDMNotFound as e:
print(str(e))
return 6

View File

@ -2556,3 +2556,7 @@ class EphemeralEncryptionSecretNotFound(Invalid):
class EphemeralEncryptionCleanupFailed(NovaException):
msg_fmt = _("Failed to clean up ephemeral encryption secrets: "
"%(error)s")
class HostConflict(Exception):
pass

View File

@ -3642,6 +3642,50 @@ class VolumeAttachmentCommandsTestCase(test.NoDBTestCase):
mock_action_start.assert_called_once()
mock_action.finish.assert_called_once()
@mock.patch('nova.compute.rpcapi.ComputeAPI', autospec=True)
@mock.patch('nova.volume.cinder.API', autospec=True)
@mock.patch('nova.compute.api.API', autospec=True)
@mock.patch.object(objects.BlockDeviceMapping, 'save')
@mock.patch.object(
objects.BlockDeviceMapping, 'get_by_volume_and_instance')
@mock.patch.object(objects.Instance, 'get_by_uuid')
@mock.patch.object(objects.InstanceAction, 'action_start')
def test_refresh_invalid_connector_host(
self, mock_action_start, mock_get_instance,
mock_get_bdm, mock_save_bdm, mock_compute_api, mock_volume_api,
mock_compute_rpcapi
):
"""Test refresh with a old host not disconnected properly
and connector host info is not correct, a fake-host is
passed.
"""
fake_volume_api = mock_volume_api.return_value
device_name = '/dev/vda'
mock_get_instance.return_value = objects.Instance(
uuid=uuidsentinel.instance,
vm_state=obj_fields.InstanceState.STOPPED,
host='old-host', locked=False)
mock_get_bdm.return_value = objects.BlockDeviceMapping(
uuid=uuidsentinel.bdm, volume_id=uuidsentinel.volume,
attachment_id=uuidsentinel.instance,
device_name=device_name)
mock_action = mock.Mock(spec=objects.InstanceAction)
mock_action_start.return_value = mock_action
fake_volume_api.attachment_create.return_value = {
'id': uuidsentinel.new_attachment,
}
# in instance we have host as 'old-host'
# but here 'fake-host' is passed in connector info.
fake_volume_api.attachment_update.return_value = {
'connection_info': self._get_fake_connector_info(),
}
ret = self._test_refresh()
self.assertEqual(7, ret)
@mock.patch('nova.compute.rpcapi.ComputeAPI', autospec=True)
@mock.patch('nova.volume.cinder.API', autospec=True)
@mock.patch('nova.compute.api.API', autospec=True)
@ -3664,7 +3708,7 @@ class VolumeAttachmentCommandsTestCase(test.NoDBTestCase):
mock_get_instance.return_value = objects.Instance(
uuid=uuidsentinel.instance,
vm_state=obj_fields.InstanceState.STOPPED,
host='foo', locked=False)
host='fake-host', locked=False)
mock_get_bdm.return_value = objects.BlockDeviceMapping(
uuid=uuidsentinel.bdm, volume_id=uuidsentinel.volume,
attachment_id=uuidsentinel.instance,