Fix live-migration failure in FC multipath case
Currently, /dev/dm-<NUM> instead of /dev/mapper/<multipath_id> is
used to access multipath FC volumes by Compute Node and
multipath_id in connection_info is not maintained properly and
may be lost during connection refreshing.
This implementation will make source Compute Node and destination
Compute Node fail to disconnect/connect to volumes properly and
result in live-migration failure.
To fix it, /dev/mapper<multipath_id> will be used instead of
/dev/dm-<NUM> to access multipath devices, just like iSCSI multipath
implementation, and logic to preserve the unique (across Compute
Nodes) multipath_id is also added.
Closes-Bug: #1327497
(cherry picked from commit 3ea14e8a70
)
Conflicts:
nova/storage/linuxscsi.py
nova/tests/virt/libvirt/test_libvirt_volume.py
nova/virt/block_device.py
nova/virt/libvirt/volume.py
This backport commit adjust oslo.i18n usage to oslo-incubator common code,
due to we didn't have oslo.i18n in icehouse.
And remove unused variable value dev_str in test_libvirt_volume.py,
it should be deleted but not worth a specific commit in stable/icehouse.
Change-Id: I17f15852c098af88afd270084c62eb87693c60d4
This commit is contained in:
parent
b61aa4d098
commit
74e0ba7e65
|
@ -4596,7 +4596,7 @@ class ComputeManager(manager.Manager):
|
||||||
|
|
||||||
# Cleanup source host post live-migration
|
# Cleanup source host post live-migration
|
||||||
block_device_info = self._get_instance_block_device_info(
|
block_device_info = self._get_instance_block_device_info(
|
||||||
ctxt, instance, bdms)
|
ctxt, instance, bdms=bdms)
|
||||||
self.driver.post_live_migration(ctxt, instance, block_device_info,
|
self.driver.post_live_migration(ctxt, instance, block_device_info,
|
||||||
migrate_data)
|
migrate_data)
|
||||||
|
|
||||||
|
|
|
@ -15,6 +15,7 @@
|
||||||
"""Generic linux scsi subsystem utilities."""
|
"""Generic linux scsi subsystem utilities."""
|
||||||
|
|
||||||
from nova.openstack.common.gettextutils import _
|
from nova.openstack.common.gettextutils import _
|
||||||
|
from nova.openstack.common.gettextutils import _LW
|
||||||
from nova.openstack.common import log as logging
|
from nova.openstack.common import log as logging
|
||||||
from nova.openstack.common import loopingcall
|
from nova.openstack.common import loopingcall
|
||||||
from nova.openstack.common import processutils
|
from nova.openstack.common import processutils
|
||||||
|
@ -97,7 +98,7 @@ def find_multipath_device(device):
|
||||||
(out, err) = utils.execute('multipath', '-l', device,
|
(out, err) = utils.execute('multipath', '-l', device,
|
||||||
run_as_root=True)
|
run_as_root=True)
|
||||||
except processutils.ProcessExecutionError as exc:
|
except processutils.ProcessExecutionError as exc:
|
||||||
LOG.warn(_("Multipath call failed exit (%(code)s)")
|
LOG.warn(_LW("Multipath call failed exit (%(code)s)")
|
||||||
% {'code': exc.exit_code})
|
% {'code': exc.exit_code})
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@ -110,15 +111,15 @@ def find_multipath_device(device):
|
||||||
# device line output is different depending
|
# device line output is different depending
|
||||||
# on /etc/multipath.conf settings.
|
# on /etc/multipath.conf settings.
|
||||||
if info[1][:2] == "dm":
|
if info[1][:2] == "dm":
|
||||||
mdev = "/dev/%s" % info[1]
|
|
||||||
mdev_id = info[0]
|
mdev_id = info[0]
|
||||||
|
mdev = '/dev/mapper/%s' % mdev_id
|
||||||
elif info[2][:2] == "dm":
|
elif info[2][:2] == "dm":
|
||||||
mdev = "/dev/%s" % info[2]
|
|
||||||
mdev_id = info[1].replace('(', '')
|
mdev_id = info[1].replace('(', '')
|
||||||
mdev_id = mdev_id.replace(')', '')
|
mdev_id = mdev_id.replace(')', '')
|
||||||
|
mdev = '/dev/mapper/%s' % mdev_id
|
||||||
|
|
||||||
if mdev is None:
|
if mdev is None:
|
||||||
LOG.warn(_("Couldn't find multipath device %s"), line)
|
LOG.warn(_LW("Couldn't find multipath device %s"), line)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
LOG.debug(_("Found multipath device = %s"), mdev)
|
LOG.debug(_("Found multipath device = %s"), mdev)
|
||||||
|
@ -126,6 +127,11 @@ def find_multipath_device(device):
|
||||||
for dev_line in device_lines:
|
for dev_line in device_lines:
|
||||||
if dev_line.find("policy") != -1:
|
if dev_line.find("policy") != -1:
|
||||||
continue
|
continue
|
||||||
|
if '#' in dev_line:
|
||||||
|
LOG.warn(_LW('Skip faulty line "%(dev_line)s" of'
|
||||||
|
' multipath device %(mdev)s')
|
||||||
|
% {'mdev': mdev, 'dev_line': dev_line})
|
||||||
|
continue
|
||||||
|
|
||||||
dev_line = dev_line.lstrip(' |-`')
|
dev_line = dev_line.lstrip(' |-`')
|
||||||
dev_info = dev_line.split()
|
dev_info = dev_line.split()
|
||||||
|
|
|
@ -59,7 +59,7 @@ class StorageLinuxSCSITestCase(test.NoDBTestCase):
|
||||||
|
|
||||||
info = linuxscsi.find_multipath_device('/dev/sde')
|
info = linuxscsi.find_multipath_device('/dev/sde')
|
||||||
LOG.error("info = %s" % info)
|
LOG.error("info = %s" % info)
|
||||||
self.assertEqual("/dev/dm-3", info["device"])
|
self.assertEqual("/dev/mapper/350002ac20398383d", info["device"])
|
||||||
self.assertEqual("/dev/sde", info['devices'][0]['device'])
|
self.assertEqual("/dev/sde", info['devices'][0]['device'])
|
||||||
self.assertEqual("0", info['devices'][0]['host'])
|
self.assertEqual("0", info['devices'][0]['host'])
|
||||||
self.assertEqual("0", info['devices'][0]['id'])
|
self.assertEqual("0", info['devices'][0]['id'])
|
||||||
|
@ -90,7 +90,8 @@ class StorageLinuxSCSITestCase(test.NoDBTestCase):
|
||||||
|
|
||||||
info = linuxscsi.find_multipath_device('/dev/sde')
|
info = linuxscsi.find_multipath_device('/dev/sde')
|
||||||
LOG.error("info = %s" % info)
|
LOG.error("info = %s" % info)
|
||||||
self.assertEqual("/dev/dm-2", info["device"])
|
self.assertEqual("/dev/mapper/36005076da00638089c000000000004d5",
|
||||||
|
info["device"])
|
||||||
self.assertEqual("/dev/sde", info['devices'][0]['device'])
|
self.assertEqual("/dev/sde", info['devices'][0]['device'])
|
||||||
self.assertEqual("6", info['devices'][0]['host'])
|
self.assertEqual("6", info['devices'][0]['host'])
|
||||||
self.assertEqual("0", info['devices'][0]['channel'])
|
self.assertEqual("0", info['devices'][0]['channel'])
|
||||||
|
@ -118,7 +119,8 @@ class StorageLinuxSCSITestCase(test.NoDBTestCase):
|
||||||
|
|
||||||
info = linuxscsi.find_multipath_device('/dev/sdd')
|
info = linuxscsi.find_multipath_device('/dev/sdd')
|
||||||
LOG.error("info = %s" % info)
|
LOG.error("info = %s" % info)
|
||||||
self.assertEqual("/dev/dm-2", info["device"])
|
self.assertEqual("/dev/mapper/36005076303ffc48e0000000000000101",
|
||||||
|
info["device"])
|
||||||
self.assertEqual("/dev/sdd", info['devices'][0]['device'])
|
self.assertEqual("/dev/sdd", info['devices'][0]['device'])
|
||||||
self.assertEqual("6", info['devices'][0]['host'])
|
self.assertEqual("6", info['devices'][0]['host'])
|
||||||
self.assertEqual("0", info['devices'][0]['channel'])
|
self.assertEqual("0", info['devices'][0]['channel'])
|
||||||
|
|
|
@ -896,15 +896,22 @@ class LibvirtVolumeTestCase(test.NoDBTestCase):
|
||||||
mount_device = "vde"
|
mount_device = "vde"
|
||||||
conf = libvirt_driver.connect_volume(connection_info,
|
conf = libvirt_driver.connect_volume(connection_info,
|
||||||
self.disk_info)
|
self.disk_info)
|
||||||
|
self.assertEqual('1234567890',
|
||||||
|
connection_info['data']['multipath_id'])
|
||||||
tree = conf.format_dom()
|
tree = conf.format_dom()
|
||||||
dev_str = '/dev/disk/by-path/pci-0000:05:00.2-fc-0x%s-lun-1' % wwn
|
self.assertEqual('block', tree.get('type'))
|
||||||
self.assertEqual(tree.get('type'), 'block')
|
self.assertEqual(multipath_devname,
|
||||||
self.assertEqual(tree.find('./source').get('dev'),
|
tree.find('./source').get('dev'))
|
||||||
multipath_devname)
|
# Test the scenario where multipath_id is returned
|
||||||
connection_info["data"]["devices"] = devices["devices"]
|
|
||||||
libvirt_driver.disconnect_volume(connection_info, mount_device)
|
libvirt_driver.disconnect_volume(connection_info, mount_device)
|
||||||
expected_commands = []
|
expected_commands = []
|
||||||
self.assertEqual(self.executes, expected_commands)
|
self.assertEqual(expected_commands, self.executes)
|
||||||
|
# Test the scenario where multipath_id is not returned
|
||||||
|
connection_info["data"]["devices"] = devices["devices"]
|
||||||
|
del connection_info["data"]["multipath_id"]
|
||||||
|
libvirt_driver.disconnect_volume(connection_info, mount_device)
|
||||||
|
expected_commands = []
|
||||||
|
self.assertEqual(expected_commands, self.executes)
|
||||||
|
|
||||||
# Should not work for anything other than string, unicode, and list
|
# Should not work for anything other than string, unicode, and list
|
||||||
connection_info = self.fibrechan_connection(self.vol,
|
connection_info = self.fibrechan_connection(self.vol,
|
||||||
|
|
|
@ -415,8 +415,8 @@ class TestDriverBlockDevice(test.NoDBTestCase):
|
||||||
|
|
||||||
instance = {'id': 'fake_id', 'uuid': 'fake_uuid'}
|
instance = {'id': 'fake_id', 'uuid': 'fake_uuid'}
|
||||||
connector = {'ip': 'fake_ip', 'host': 'fake_host'}
|
connector = {'ip': 'fake_ip', 'host': 'fake_host'}
|
||||||
connection_info = {'data': {}}
|
connection_info = {'data': {'multipath_id': 'fake_multipath_id'}}
|
||||||
expected_conn_info = {'data': {},
|
expected_conn_info = {'data': {'multipath_id': 'fake_multipath_id'},
|
||||||
'serial': 'fake-volume-id-2'}
|
'serial': 'fake-volume-id-2'}
|
||||||
|
|
||||||
self.mox.StubOutWithMock(test_bdm._bdm_obj, 'save')
|
self.mox.StubOutWithMock(test_bdm._bdm_obj, 'save')
|
||||||
|
|
|
@ -19,6 +19,7 @@ from nova import block_device
|
||||||
from nova.objects import block_device as block_device_obj
|
from nova.objects import block_device as block_device_obj
|
||||||
from nova.openstack.common import excutils
|
from nova.openstack.common import excutils
|
||||||
from nova.openstack.common.gettextutils import _
|
from nova.openstack.common.gettextutils import _
|
||||||
|
from nova.openstack.common.gettextutils import _LI
|
||||||
from nova.openstack.common import jsonutils
|
from nova.openstack.common import jsonutils
|
||||||
from nova.openstack.common import log as logging
|
from nova.openstack.common import log as logging
|
||||||
from nova.volume import encryptors
|
from nova.volume import encryptors
|
||||||
|
@ -209,6 +210,14 @@ class DriverVolumeBlockDevice(DriverBlockDevice):
|
||||||
except TypeError:
|
except TypeError:
|
||||||
self['connection_info'] = None
|
self['connection_info'] = None
|
||||||
|
|
||||||
|
def _preserve_multipath_id(self, connection_info):
|
||||||
|
if self['connection_info'] and 'data' in self['connection_info']:
|
||||||
|
if 'multipath_id' in self['connection_info']['data']:
|
||||||
|
connection_info['data']['multipath_id'] =\
|
||||||
|
self['connection_info']['data']['multipath_id']
|
||||||
|
LOG.info(_LI('preserve multipath_id %s'),
|
||||||
|
connection_info['data']['multipath_id'])
|
||||||
|
|
||||||
@update_db
|
@update_db
|
||||||
def attach(self, context, instance, volume_api, virt_driver,
|
def attach(self, context, instance, volume_api, virt_driver,
|
||||||
do_check_attach=True, do_driver_attach=False):
|
do_check_attach=True, do_driver_attach=False):
|
||||||
|
@ -225,6 +234,7 @@ class DriverVolumeBlockDevice(DriverBlockDevice):
|
||||||
connector)
|
connector)
|
||||||
if 'serial' not in connection_info:
|
if 'serial' not in connection_info:
|
||||||
connection_info['serial'] = self.volume_id
|
connection_info['serial'] = self.volume_id
|
||||||
|
self._preserve_multipath_id(connection_info)
|
||||||
|
|
||||||
# If do_driver_attach is False, we will attach a volume to an instance
|
# If do_driver_attach is False, we will attach a volume to an instance
|
||||||
# at boot time. So actual attach is done by instance creation code.
|
# at boot time. So actual attach is done by instance creation code.
|
||||||
|
@ -267,6 +277,7 @@ class DriverVolumeBlockDevice(DriverBlockDevice):
|
||||||
connector)
|
connector)
|
||||||
if 'serial' not in connection_info:
|
if 'serial' not in connection_info:
|
||||||
connection_info['serial'] = self.volume_id
|
connection_info['serial'] = self.volume_id
|
||||||
|
self._preserve_multipath_id(connection_info)
|
||||||
self['connection_info'] = connection_info
|
self['connection_info'] = connection_info
|
||||||
|
|
||||||
def save(self, context):
|
def save(self, context):
|
||||||
|
|
|
@ -27,6 +27,7 @@ import six.moves.urllib.parse as urlparse
|
||||||
|
|
||||||
from nova import exception
|
from nova import exception
|
||||||
from nova.openstack.common.gettextutils import _
|
from nova.openstack.common.gettextutils import _
|
||||||
|
from nova.openstack.common.gettextutils import _LW
|
||||||
from nova.openstack.common import log as logging
|
from nova.openstack.common import log as logging
|
||||||
from nova.openstack.common import loopingcall
|
from nova.openstack.common import loopingcall
|
||||||
from nova.openstack.common import processutils
|
from nova.openstack.common import processutils
|
||||||
|
@ -993,7 +994,6 @@ class LibvirtFibreChannelVolumeDriver(LibvirtBaseVolumeDriver):
|
||||||
"""Detach the volume from instance_name."""
|
"""Detach the volume from instance_name."""
|
||||||
super(LibvirtFibreChannelVolumeDriver,
|
super(LibvirtFibreChannelVolumeDriver,
|
||||||
self).disconnect_volume(connection_info, mount_device)
|
self).disconnect_volume(connection_info, mount_device)
|
||||||
devices = connection_info['data']['devices']
|
|
||||||
|
|
||||||
# If this is a multipath device, we need to search again
|
# If this is a multipath device, we need to search again
|
||||||
# and make sure we remove all the devices. Some of them
|
# and make sure we remove all the devices. Some of them
|
||||||
|
@ -1003,6 +1003,11 @@ class LibvirtFibreChannelVolumeDriver(LibvirtBaseVolumeDriver):
|
||||||
mdev_info = linuxscsi.find_multipath_device(multipath_id)
|
mdev_info = linuxscsi.find_multipath_device(multipath_id)
|
||||||
devices = mdev_info['devices']
|
devices = mdev_info['devices']
|
||||||
LOG.debug(_("devices to remove = %s"), devices)
|
LOG.debug(_("devices to remove = %s"), devices)
|
||||||
|
else:
|
||||||
|
# only needed when multipath-tools work improperly
|
||||||
|
devices = connection_info['data'].get('devices', [])
|
||||||
|
LOG.warn(_LW("multipath-tools probably work improperly. "
|
||||||
|
"devices to remove = %s.") % devices)
|
||||||
|
|
||||||
# There may have been more than 1 device mounted
|
# There may have been more than 1 device mounted
|
||||||
# by the kernel for this volume. We have to remove
|
# by the kernel for this volume. We have to remove
|
||||||
|
|
2
tox.ini
2
tox.ini
|
@ -58,4 +58,4 @@ exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build,t
|
||||||
|
|
||||||
[hacking]
|
[hacking]
|
||||||
local-check-factory = nova.hacking.checks.factory
|
local-check-factory = nova.hacking.checks.factory
|
||||||
import_exceptions = nova.openstack.common.gettextutils._
|
import_exceptions = nova.openstack.common.gettextutils
|
||||||
|
|
Loading…
Reference in New Issue