Add NVMeoF Multipathing support
Phase 1 (Native) implementation of NVMeoF Multipathing. Change-Id: I3af33c5e43cfb104e436fb785b08fb28b50a031a
This commit is contained in:
parent
56bf0272b5
commit
03a5af79fe
|
@ -43,6 +43,8 @@ LOG = logging.getLogger(__name__)
|
||||||
class NVMeOFConnector(base.BaseLinuxConnector):
|
class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
"""Connector class to attach/detach NVMe-oF volumes."""
|
"""Connector class to attach/detach NVMe-oF volumes."""
|
||||||
|
|
||||||
|
native_multipath_supported = None
|
||||||
|
|
||||||
def __init__(self, root_helper, driver=None, use_multipath=False,
|
def __init__(self, root_helper, driver=None, use_multipath=False,
|
||||||
device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT,
|
device_scan_attempts=DEVICE_SCAN_ATTEMPTS_DEFAULT,
|
||||||
*args, **kwargs):
|
*args, **kwargs):
|
||||||
|
@ -52,6 +54,10 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
device_scan_attempts=device_scan_attempts,
|
device_scan_attempts=device_scan_attempts,
|
||||||
*args, **kwargs)
|
*args, **kwargs)
|
||||||
self.use_multipath = use_multipath
|
self.use_multipath = use_multipath
|
||||||
|
self._set_native_multipath_supported()
|
||||||
|
if self.use_multipath and not \
|
||||||
|
NVMeOFConnector.native_multipath_supported:
|
||||||
|
LOG.warning('native multipath is not enabled')
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_search_path():
|
def get_search_path():
|
||||||
|
@ -99,7 +105,6 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
execute = kwargs.get('execute') or priv_rootwrap.execute
|
execute = kwargs.get('execute') or priv_rootwrap.execute
|
||||||
nvmf = NVMeOFConnector(root_helper=root_helper, execute=execute)
|
nvmf = NVMeOFConnector(root_helper=root_helper, execute=execute)
|
||||||
ret = {}
|
ret = {}
|
||||||
|
|
||||||
nqn = None
|
nqn = None
|
||||||
uuid = nvmf._get_host_uuid()
|
uuid = nvmf._get_host_uuid()
|
||||||
suuid = nvmf._get_system_uuid()
|
suuid = nvmf._get_system_uuid()
|
||||||
|
@ -111,6 +116,7 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
ret['system uuid'] = suuid # compatibility
|
ret['system uuid'] = suuid # compatibility
|
||||||
if nqn:
|
if nqn:
|
||||||
ret['nqn'] = nqn
|
ret['nqn'] = nqn
|
||||||
|
ret['nvme_native_multipath'] = cls._set_native_multipath_supported()
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def _get_host_uuid(self):
|
def _get_host_uuid(self):
|
||||||
|
@ -148,6 +154,22 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
out = ""
|
out = ""
|
||||||
return out.strip()
|
return out.strip()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _set_native_multipath_supported(cls):
|
||||||
|
if cls.native_multipath_supported is None:
|
||||||
|
cls.native_multipath_supported = \
|
||||||
|
cls._is_native_multipath_supported()
|
||||||
|
return cls.native_multipath_supported
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_native_multipath_supported():
|
||||||
|
try:
|
||||||
|
with open('/sys/module/nvme_core/parameters/multipath', 'rt') as f:
|
||||||
|
return f.read().strip() == 'Y'
|
||||||
|
except Exception:
|
||||||
|
LOG.warning("Could not find nvme_core/parameters/multipath")
|
||||||
|
return False
|
||||||
|
|
||||||
def _get_nvme_devices(self):
|
def _get_nvme_devices(self):
|
||||||
nvme_devices = []
|
nvme_devices = []
|
||||||
# match nvme devices like /dev/nvme10n10
|
# match nvme devices like /dev/nvme10n10
|
||||||
|
@ -321,7 +343,7 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
:returns: dict
|
:returns: dict
|
||||||
"""
|
"""
|
||||||
if connection_properties.get('vol_uuid'): # compatibility
|
if connection_properties.get('vol_uuid'): # compatibility
|
||||||
return self._connect_volume_replicated(connection_properties)
|
return self._connect_volume_by_uuid(connection_properties)
|
||||||
|
|
||||||
current_nvme_devices = self._get_nvme_devices()
|
current_nvme_devices = self._get_nvme_devices()
|
||||||
device_info = {'type': 'block'}
|
device_info = {'type': 'block'}
|
||||||
|
@ -425,7 +447,7 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
raise exception.VolumePathsNotFound()
|
raise exception.VolumePathsNotFound()
|
||||||
|
|
||||||
@utils.trace
|
@utils.trace
|
||||||
def _connect_volume_replicated(self, connection_properties):
|
def _connect_volume_by_uuid(self, connection_properties):
|
||||||
"""connect to volume on host
|
"""connect to volume on host
|
||||||
|
|
||||||
connection_properties for NVMe-oF must include:
|
connection_properties for NVMe-oF must include:
|
||||||
|
@ -433,7 +455,6 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
target_nqn - NVMe-oF Qualified Name
|
target_nqn - NVMe-oF Qualified Name
|
||||||
vol_uuid - UUID for volume/replica
|
vol_uuid - UUID for volume/replica
|
||||||
"""
|
"""
|
||||||
|
|
||||||
volume_replicas = connection_properties.get('volume_replicas')
|
volume_replicas = connection_properties.get('volume_replicas')
|
||||||
replica_count = connection_properties.get('replica_count')
|
replica_count = connection_properties.get('replica_count')
|
||||||
volume_alias = connection_properties.get('alias')
|
volume_alias = connection_properties.get('alias')
|
||||||
|
@ -499,6 +520,8 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
NVMeOFConnector.run_mdadm(
|
NVMeOFConnector.run_mdadm(
|
||||||
self, ['mdadm', '--grow', '--size', 'max', device_path])
|
self, ['mdadm', '--grow', '--size', 'max', device_path])
|
||||||
else:
|
else:
|
||||||
|
target_nqn = None
|
||||||
|
vol_uuid = None
|
||||||
if not volume_replicas:
|
if not volume_replicas:
|
||||||
target_nqn = connection_properties['target_nqn']
|
target_nqn = connection_properties['target_nqn']
|
||||||
vol_uuid = connection_properties['vol_uuid']
|
vol_uuid = connection_properties['vol_uuid']
|
||||||
|
@ -511,28 +534,43 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
return self._linuxscsi.get_device_size(device_path)
|
return self._linuxscsi.get_device_size(device_path)
|
||||||
|
|
||||||
def _connect_target_volume(self, target_nqn, vol_uuid, portals):
|
def _connect_target_volume(self, target_nqn, vol_uuid, portals):
|
||||||
try:
|
nvme_ctrls = NVMeOFConnector.rescan(self, target_nqn)
|
||||||
NVMeOFConnector._get_nvme_controller(self, target_nqn)
|
any_new_connect = NVMeOFConnector.connect_to_portals(self, target_nqn,
|
||||||
NVMeOFConnector.rescan(self, target_nqn, vol_uuid)
|
portals,
|
||||||
except exception.VolumeDeviceNotFound:
|
nvme_ctrls)
|
||||||
if not NVMeOFConnector.connect_to_portals(
|
if not any_new_connect and len(nvme_ctrls) == 0:
|
||||||
self, target_nqn, portals):
|
# no new connections and any pre-exists controllers
|
||||||
LOG.error("No successful connections to: %s", target_nqn)
|
LOG.error("No successful connections to: %s", target_nqn)
|
||||||
raise exception.VolumeDeviceNotFound(device=target_nqn)
|
raise exception.VolumeDeviceNotFound(device=target_nqn)
|
||||||
dev_path = NVMeOFConnector.get_nvme_device_path(
|
if any_new_connect:
|
||||||
self, target_nqn, vol_uuid)
|
# new connections - refresh controllers map
|
||||||
|
nvme_ctrls = \
|
||||||
|
NVMeOFConnector.get_live_nvme_controllers_map(self, target_nqn)
|
||||||
|
nvme_ctrls_values = list(nvme_ctrls.values())
|
||||||
|
dev_path = NVMeOFConnector.get_nvme_device_path(self, target_nqn,
|
||||||
|
vol_uuid,
|
||||||
|
nvme_ctrls_values)
|
||||||
if not dev_path:
|
if not dev_path:
|
||||||
LOG.error("Target %s volume %s not found", target_nqn, vol_uuid)
|
LOG.error("Target %s volume %s not found", target_nqn, vol_uuid)
|
||||||
raise exception.VolumeDeviceNotFound(device=vol_uuid)
|
raise exception.VolumeDeviceNotFound(device=vol_uuid)
|
||||||
return dev_path
|
return dev_path
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def connect_to_portals(executor, target_nqn, target_portals):
|
def connect_to_portals(executor, target_nqn, target_portals, nvme_ctrls):
|
||||||
"""connect to any of NVMe-oF target portals"""
|
# connect to any of NVMe-oF target portals -
|
||||||
any_connect = False
|
# check if the controller exist before trying to connect
|
||||||
|
# in multipath connect all given target portals
|
||||||
|
any_new_connect = False
|
||||||
|
no_multipath = not executor.use_multipath or not \
|
||||||
|
NVMeOFConnector.native_multipath_supported
|
||||||
for portal in target_portals:
|
for portal in target_portals:
|
||||||
portal_address = portal[0]
|
portal_address = portal[0]
|
||||||
portal_port = portal[1]
|
portal_port = portal[1]
|
||||||
|
if NVMeOFConnector.is_portal_connected(portal_address, portal_port,
|
||||||
|
nvme_ctrls):
|
||||||
|
if no_multipath:
|
||||||
|
break
|
||||||
|
continue
|
||||||
if portal[2] == 'RoCEv2':
|
if portal[2] == 'RoCEv2':
|
||||||
portal_transport = 'rdma'
|
portal_transport = 'rdma'
|
||||||
else:
|
else:
|
||||||
|
@ -542,14 +580,30 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
portal_transport, '-n', target_nqn, '-Q', '128', '-l', '-1')
|
portal_transport, '-n', target_nqn, '-Q', '128', '-l', '-1')
|
||||||
try:
|
try:
|
||||||
NVMeOFConnector.run_nvme_cli(executor, nvme_command)
|
NVMeOFConnector.run_nvme_cli(executor, nvme_command)
|
||||||
any_connect = True
|
any_new_connect = True
|
||||||
break
|
if no_multipath:
|
||||||
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception("Could not connect to portal %s", portal)
|
LOG.exception("Could not connect to portal %s", portal)
|
||||||
return any_connect
|
return any_new_connect
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_nvme_controller(executor, target_nqn):
|
def is_portal_connected(portal_address, portal_port, nvme_ctrls):
|
||||||
|
address = f"traddr={portal_address},trsvcid={portal_port}"
|
||||||
|
return address in nvme_ctrls
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_nvme_controllers(executor, target_nqn):
|
||||||
|
nvme_controllers = \
|
||||||
|
NVMeOFConnector.get_live_nvme_controllers_map(executor, target_nqn)
|
||||||
|
if len(nvme_controllers) > 0:
|
||||||
|
return nvme_controllers.values()
|
||||||
|
raise exception.VolumeDeviceNotFound(device=target_nqn)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_live_nvme_controllers_map(executor, target_nqn):
|
||||||
|
"""returns map of all live controllers and their addresses """
|
||||||
|
nvme_controllers = dict()
|
||||||
ctrls = glob.glob('/sys/class/nvme-fabrics/ctl/nvme*')
|
ctrls = glob.glob('/sys/class/nvme-fabrics/ctl/nvme*')
|
||||||
for ctrl in ctrls:
|
for ctrl in ctrls:
|
||||||
try:
|
try:
|
||||||
|
@ -561,31 +615,45 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
state, _err = executor._execute(
|
state, _err = executor._execute(
|
||||||
'cat', ctrl + '/state', run_as_root=True,
|
'cat', ctrl + '/state', run_as_root=True,
|
||||||
root_helper=executor._root_helper)
|
root_helper=executor._root_helper)
|
||||||
if 'live' not in state:
|
if 'live' in state:
|
||||||
|
address_file = ctrl + '/address'
|
||||||
|
try:
|
||||||
|
with open(address_file, 'rt') as f:
|
||||||
|
address = f.read().strip()
|
||||||
|
except Exception:
|
||||||
|
LOG.warning("Failed to read file %s",
|
||||||
|
address_file)
|
||||||
|
continue
|
||||||
|
ctrl_name = os.path.basename(ctrl)
|
||||||
|
LOG.debug("[!] address: %s|%s", address, ctrl_name)
|
||||||
|
nvme_controllers[address] = ctrl_name
|
||||||
|
else:
|
||||||
LOG.debug("nvmeof ctrl device not live: %s", ctrl)
|
LOG.debug("nvmeof ctrl device not live: %s", ctrl)
|
||||||
raise exception.VolumeDeviceNotFound(device=ctrl)
|
|
||||||
return ctrl[ctrl.rfind('/') + 1:]
|
|
||||||
except putils.ProcessExecutionError as e:
|
except putils.ProcessExecutionError as e:
|
||||||
LOG.exception(e)
|
LOG.exception(e)
|
||||||
|
return nvme_controllers
|
||||||
raise exception.VolumeDeviceNotFound(device=target_nqn)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@utils.retry(exception.VolumeDeviceNotFound, retries=5)
|
@utils.retry(exception.VolumeDeviceNotFound, retries=5)
|
||||||
def get_nvme_device_path(executor, target_nqn, vol_uuid):
|
def get_nvme_device_path(executor, target_nqn, vol_uuid, nvme_ctrls=None):
|
||||||
nvme_ctrl = NVMeOFConnector._get_nvme_controller(executor, target_nqn)
|
if not nvme_ctrls:
|
||||||
uuid_paths = glob.glob('/sys/class/block/' + nvme_ctrl + 'n*/uuid')
|
nvme_ctrls = NVMeOFConnector.get_nvme_controllers(executor,
|
||||||
for uuid_path in uuid_paths:
|
target_nqn)
|
||||||
try:
|
LOG.debug("[!] nvme_ctrls: %s", nvme_ctrls)
|
||||||
uuid_lines, _err = executor._execute(
|
for nvme_ctrl in nvme_ctrls:
|
||||||
'cat', uuid_path, run_as_root=True,
|
uuid_paths = glob.glob('/sys/class/block/' + nvme_ctrl + 'n*/uuid')
|
||||||
root_helper=executor._root_helper)
|
for uuid_path in uuid_paths:
|
||||||
if uuid_lines.split('\n')[0] == vol_uuid:
|
try:
|
||||||
ignore = len('/uuid')
|
uuid_lines, _err = executor._execute(
|
||||||
return '/dev/' + uuid_path[
|
'cat', uuid_path, run_as_root=True,
|
||||||
uuid_path.rfind('/', 0, -ignore) + 1: -ignore]
|
root_helper=executor._root_helper)
|
||||||
except putils.ProcessExecutionError as e:
|
if uuid_lines.split('\n')[0] == vol_uuid:
|
||||||
LOG.exception(e)
|
ignore = len('/uuid')
|
||||||
|
ns_ind = uuid_path.rfind('/', 0, -ignore)
|
||||||
|
nvme_device = uuid_path[ns_ind + 1: -ignore]
|
||||||
|
return '/dev/' + nvme_device
|
||||||
|
except putils.ProcessExecutionError as e:
|
||||||
|
LOG.exception(e)
|
||||||
raise exception.VolumeDeviceNotFound(device=vol_uuid)
|
raise exception.VolumeDeviceNotFound(device=vol_uuid)
|
||||||
|
|
||||||
def _handle_replicated_volume(self, host_device_paths,
|
def _handle_replicated_volume(self, host_device_paths,
|
||||||
|
@ -758,7 +826,7 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
|
|
||||||
LOG.debug('[!] cmd = ' + str(cmd))
|
LOG.debug('[!] cmd = ' + str(cmd))
|
||||||
NVMeOFConnector.run_mdadm(executor, cmd)
|
NVMeOFConnector.run_mdadm(executor, cmd)
|
||||||
# sometimes under load, md is not created right away so we wait
|
# sometimes under load, md is not created right away, so we wait
|
||||||
for i in range(60):
|
for i in range(60):
|
||||||
try:
|
try:
|
||||||
is_exist = os.path.exists("/dev/md/" + name)
|
is_exist = os.path.exists("/dev/md/" + name)
|
||||||
|
@ -836,15 +904,17 @@ class NVMeOFConnector(base.BaseLinuxConnector):
|
||||||
return out, err
|
return out, err
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def rescan(executor, target_nqn, vol_uuid):
|
def rescan(executor, target_nqn):
|
||||||
ctr_device = (
|
nvme_ctrls = NVMeOFConnector.get_live_nvme_controllers_map(executor,
|
||||||
NVMeOFConnector.get_search_path() +
|
target_nqn)
|
||||||
NVMeOFConnector._get_nvme_controller(executor, target_nqn))
|
for nvme_ctrl in nvme_ctrls.values():
|
||||||
nvme_command = ('ns-rescan', ctr_device)
|
ctr_device = (NVMeOFConnector.get_search_path() + nvme_ctrl)
|
||||||
try:
|
nvme_command = ('ns-rescan', ctr_device)
|
||||||
NVMeOFConnector.run_nvme_cli(executor, nvme_command)
|
try:
|
||||||
except Exception as e:
|
NVMeOFConnector.run_nvme_cli(executor, nvme_command)
|
||||||
raise exception.CommandExecutionFailed(e, cmd=nvme_command)
|
except Exception as e:
|
||||||
|
LOG.exception(e)
|
||||||
|
return nvme_ctrls
|
||||||
|
|
||||||
def _get_fs_type(self, device_path):
|
def _get_fs_type(self, device_path):
|
||||||
cmd = ['blkid', device_path, '-s', 'TYPE', '-o', 'value']
|
cmd = ['blkid', device_path, '-s', 'TYPE', '-o', 'value']
|
||||||
|
|
|
@ -50,7 +50,8 @@ connection_properties = {
|
||||||
'replica_count': 3
|
'replica_count': 3
|
||||||
}
|
}
|
||||||
fake_portal = ('fake', 'portal', 'tcp')
|
fake_portal = ('fake', 'portal', 'tcp')
|
||||||
|
fake_controller = '/sys/class/nvme-fabrics/ctl/nvme1'
|
||||||
|
fake_controllers_map = {'traddr=fakeaddress,trsvcid=4430': 'nvme1'}
|
||||||
nvme_list_subsystems_stdout = """
|
nvme_list_subsystems_stdout = """
|
||||||
{
|
{
|
||||||
"Subsystems" : [
|
"Subsystems" : [
|
||||||
|
@ -137,6 +138,9 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
uuid = self.connector._get_host_uuid()
|
uuid = self.connector._get_host_uuid()
|
||||||
self.assertIsNone(uuid)
|
self.assertIsNone(uuid)
|
||||||
|
|
||||||
|
@mock.patch.object(nvmeof.NVMeOFConnector,
|
||||||
|
'_is_native_multipath_supported',
|
||||||
|
return_value=True)
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'nvme_present',
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'nvme_present',
|
||||||
return_value=True)
|
return_value=True)
|
||||||
@mock.patch.object(utils, 'get_host_nqn',
|
@mock.patch.object(utils, 'get_host_nqn',
|
||||||
|
@ -147,11 +151,15 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
return_value=None)
|
return_value=None)
|
||||||
def test_get_connector_properties_without_sysuuid(self, mock_host_uuid,
|
def test_get_connector_properties_without_sysuuid(self, mock_host_uuid,
|
||||||
mock_sysuuid, mock_nqn,
|
mock_sysuuid, mock_nqn,
|
||||||
mock_nvme_present):
|
mock_nvme_present,
|
||||||
|
mock_nat_mpath_support):
|
||||||
props = self.connector.get_connector_properties('sudo')
|
props = self.connector.get_connector_properties('sudo')
|
||||||
expected_props = {'nqn': 'fakenqn'}
|
expected_props = {'nqn': 'fakenqn', 'nvme_native_multipath': False}
|
||||||
self.assertEqual(expected_props, props)
|
self.assertEqual(expected_props, props)
|
||||||
|
|
||||||
|
@mock.patch.object(nvmeof.NVMeOFConnector,
|
||||||
|
'_is_native_multipath_supported',
|
||||||
|
return_value=True)
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'nvme_present')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'nvme_present')
|
||||||
@mock.patch.object(utils, 'get_host_nqn', autospec=True)
|
@mock.patch.object(utils, 'get_host_nqn', autospec=True)
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_system_uuid',
|
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_system_uuid',
|
||||||
|
@ -159,14 +167,15 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_host_uuid', autospec=True)
|
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_host_uuid', autospec=True)
|
||||||
def test_get_connector_properties_with_sysuuid(self, mock_host_uuid,
|
def test_get_connector_properties_with_sysuuid(self, mock_host_uuid,
|
||||||
mock_sysuuid, mock_nqn,
|
mock_sysuuid, mock_nqn,
|
||||||
mock_nvme_present):
|
mock_nvme_present,
|
||||||
|
mock_native_mpath_support):
|
||||||
mock_host_uuid.return_value = HOST_UUID
|
mock_host_uuid.return_value = HOST_UUID
|
||||||
mock_sysuuid.return_value = SYS_UUID
|
mock_sysuuid.return_value = SYS_UUID
|
||||||
mock_nqn.return_value = HOST_NQN
|
mock_nqn.return_value = HOST_NQN
|
||||||
mock_nvme_present.return_value = True
|
mock_nvme_present.return_value = True
|
||||||
props = self.connector.get_connector_properties('sudo')
|
props = self.connector.get_connector_properties('sudo')
|
||||||
expected_props = {"system uuid": SYS_UUID, "nqn": HOST_NQN,
|
expected_props = {"system uuid": SYS_UUID, "nqn": HOST_NQN,
|
||||||
"uuid": HOST_UUID}
|
"uuid": HOST_UUID, 'nvme_native_multipath': False}
|
||||||
self.assertEqual(expected_props, props)
|
self.assertEqual(expected_props, props)
|
||||||
|
|
||||||
def test_get_volume_paths_unreplicated(self):
|
def test_get_volume_paths_unreplicated(self):
|
||||||
|
@ -277,7 +286,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
self, mock_connect_target_volume):
|
self, mock_connect_target_volume):
|
||||||
mock_connect_target_volume.return_value = '/dev/nvme0n1'
|
mock_connect_target_volume.return_value = '/dev/nvme0n1'
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.connector._connect_volume_replicated(
|
self.connector._connect_volume_by_uuid(
|
||||||
{
|
{
|
||||||
'target_nqn': 'fakenqn',
|
'target_nqn': 'fakenqn',
|
||||||
'vol_uuid': 'fakeuuid',
|
'vol_uuid': 'fakeuuid',
|
||||||
|
@ -493,34 +502,30 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
)
|
)
|
||||||
mock_device_size.assert_called_with(device_path)
|
mock_device_size.assert_called_with(device_path)
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_controller')
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'rescan')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'rescan')
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_device_path')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_device_path')
|
||||||
def test__connect_target_volume_with_connected_device(
|
def test__connect_target_volume_with_connected_device(
|
||||||
self, mock_device_path, mock_rescan, mock_controller):
|
self, mock_device_path, mock_rescan):
|
||||||
mock_device_path.return_value = '/dev/nvme0n1'
|
mock_device_path.return_value = '/dev/nvme0n1'
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.connector._connect_target_volume(
|
self.connector._connect_target_volume(
|
||||||
'fakenqn', 'fakeuuid', [('fake', 'portal', 'tcp')]),
|
'fakenqn', 'fakeuuid', [('fake', 'portal', 'tcp')]),
|
||||||
'/dev/nvme0n1')
|
'/dev/nvme0n1')
|
||||||
mock_controller.assert_called_with(self.connector, 'fakenqn')
|
mock_rescan.assert_called_with(self.connector, 'fakenqn')
|
||||||
mock_rescan.assert_called_with(self.connector, 'fakenqn', 'fakeuuid')
|
|
||||||
mock_device_path.assert_called_with(
|
mock_device_path.assert_called_with(
|
||||||
self.connector, 'fakenqn', 'fakeuuid')
|
self.connector, 'fakenqn', 'fakeuuid', list({}.values()))
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'connect_to_portals')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'connect_to_portals')
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_device_path')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_device_path')
|
||||||
def test__connect_target_volume_not_connected(
|
def test__connect_target_volume_not_connected(
|
||||||
self, mock_device_path, mock_portals):
|
self, mock_device_path, mock_portals):
|
||||||
mock_device_path.side_effect = exception.VolumeDeviceNotFound()
|
mock_device_path.side_effect = exception.VolumeDeviceNotFound()
|
||||||
mock_portals.return_value = True
|
mock_portals.return_value = False
|
||||||
self.assertRaises(exception.VolumeDeviceNotFound,
|
self.assertRaises(exception.VolumeDeviceNotFound,
|
||||||
self.connector._connect_target_volume, TARGET_NQN,
|
self.connector._connect_target_volume, TARGET_NQN,
|
||||||
VOL_UUID, [('fake', 'portal', 'tcp')])
|
VOL_UUID, [('fake', 'portal', 'tcp')])
|
||||||
mock_device_path.assert_called_with(
|
|
||||||
self.connector, TARGET_NQN, VOL_UUID)
|
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_controller')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_controllers')
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'connect_to_portals')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'connect_to_portals')
|
||||||
def test__connect_target_volume_no_portals_con(
|
def test__connect_target_volume_no_portals_con(
|
||||||
self, mock_portals, mock_controller):
|
self, mock_portals, mock_controller):
|
||||||
|
@ -530,21 +535,30 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
self.connector._connect_target_volume, 'fakenqn',
|
self.connector._connect_target_volume, 'fakenqn',
|
||||||
'fakeuuid', [fake_portal])
|
'fakeuuid', [fake_portal])
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_controller')
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'connect_to_portals')
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_device_path')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_device_path')
|
||||||
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_live_nvme_controllers_map')
|
||||||
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'connect_to_portals')
|
||||||
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'rescan')
|
||||||
def test__connect_target_volume_new_device_path(
|
def test__connect_target_volume_new_device_path(
|
||||||
self, mock_device_path, mock_connect_portal, mock_controller):
|
self, mock_rescan, mock_connect_portal,
|
||||||
mock_controller.side_effect = exception.VolumeDeviceNotFound()
|
mock_get_live_nvme_controllers_map, mock_device_path):
|
||||||
mock_device_path.return_value = '/dev/nvme0n1'
|
mock_device_path.return_value = '/dev/nvme0n1'
|
||||||
|
mock_rescan.return_value = {}
|
||||||
|
mock_connect_portal.return_value = True
|
||||||
|
mock_get_live_nvme_controllers_map.return_value = fake_controllers_map
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.connector._connect_target_volume(
|
self.connector._connect_target_volume(
|
||||||
'fakenqn', 'fakeuuid', [('fake', 'portal', 'tcp')]),
|
'fakenqn', 'fakeuuid', [('fake', 'portal', 'tcp')]),
|
||||||
'/dev/nvme0n1')
|
'/dev/nvme0n1')
|
||||||
|
mock_rescan.assert_called_with(self.connector, 'fakenqn')
|
||||||
mock_connect_portal.assert_called_with(
|
mock_connect_portal.assert_called_with(
|
||||||
self.connector, 'fakenqn', [('fake', 'portal', 'tcp')])
|
self.connector, 'fakenqn', [('fake', 'portal', 'tcp')], {})
|
||||||
|
mock_get_live_nvme_controllers_map.assert_called_with(self.connector,
|
||||||
|
'fakenqn')
|
||||||
|
fake_controllers_map_values = fake_controllers_map.values()
|
||||||
mock_device_path.assert_called_with(
|
mock_device_path.assert_called_with(
|
||||||
self.connector, 'fakenqn', 'fakeuuid')
|
self.connector, 'fakenqn', 'fakeuuid',
|
||||||
|
list(fake_controllers_map_values))
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'run_nvme_cli')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'run_nvme_cli')
|
||||||
def test_connect_to_portals(self, mock_nvme_cli):
|
def test_connect_to_portals(self, mock_nvme_cli):
|
||||||
|
@ -553,7 +567,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
'tcp', '-n', 'fakenqn', '-Q', '128', '-l', '-1')
|
'tcp', '-n', 'fakenqn', '-Q', '128', '-l', '-1')
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.connector.connect_to_portals(
|
self.connector.connect_to_portals(
|
||||||
self.connector, 'fakenqn', [('10.0.0.1', 4420, 'tcp')]),
|
self.connector, 'fakenqn', [('10.0.0.1', 4420, 'tcp')], {}),
|
||||||
True)
|
True)
|
||||||
mock_nvme_cli.assert_called_with(self.connector, nvme_command)
|
mock_nvme_cli.assert_called_with(self.connector, nvme_command)
|
||||||
|
|
||||||
|
@ -565,7 +579,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
'rdma', '-n', 'fakenqn', '-Q', '128', '-l', '-1')
|
'rdma', '-n', 'fakenqn', '-Q', '128', '-l', '-1')
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
self.connector.connect_to_portals(
|
self.connector.connect_to_portals(
|
||||||
self.connector, 'fakenqn', [('10.0.0.1', 4420, 'RoCEv2')]),
|
self.connector, 'fakenqn', [('10.0.0.1', 4420, 'RoCEv2')], {}),
|
||||||
False)
|
False)
|
||||||
mock_nvme_cli.assert_called_with(self.connector, nvme_command)
|
mock_nvme_cli.assert_called_with(self.connector, nvme_command)
|
||||||
|
|
||||||
|
@ -735,25 +749,28 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
self.connector, ['mdadm', '--remove', '/dev/md/md1'])
|
self.connector, ['mdadm', '--remove', '/dev/md/md1'])
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'run_nvme_cli')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'run_nvme_cli')
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_controller')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_live_nvme_controllers_map')
|
||||||
def test_rescan(self, mock_get_nvme_controller, mock_run_nvme_cli):
|
def test_rescan(self, mock_get_live_nvme_controllers_map,
|
||||||
mock_get_nvme_controller.return_value = 'nvme1'
|
mock_run_nvme_cli):
|
||||||
|
mock_get_live_nvme_controllers_map.return_value = fake_controllers_map
|
||||||
mock_run_nvme_cli.return_value = None
|
mock_run_nvme_cli.return_value = None
|
||||||
result = self.connector.rescan(EXECUTOR, TARGET_NQN, VOL_UUID)
|
result = self.connector.rescan(EXECUTOR, TARGET_NQN)
|
||||||
self.assertIsNone(result)
|
self.assertEqual(fake_controllers_map, result)
|
||||||
mock_get_nvme_controller.assert_called_with(EXECUTOR, TARGET_NQN)
|
mock_get_live_nvme_controllers_map.assert_called_with(EXECUTOR,
|
||||||
|
TARGET_NQN)
|
||||||
nvme_command = ('ns-rescan', NVME_DEVICE_PATH)
|
nvme_command = ('ns-rescan', NVME_DEVICE_PATH)
|
||||||
mock_run_nvme_cli.assert_called_with(EXECUTOR, nvme_command)
|
mock_run_nvme_cli.assert_called_with(EXECUTOR, nvme_command)
|
||||||
|
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, 'run_nvme_cli')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'run_nvme_cli')
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_controller')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_live_nvme_controllers_map')
|
||||||
def test_rescan_err(self, mock_get_nvme_controller, mock_run_nvme_cli):
|
def test_rescan_err(self, mock_get_live_nvme_controllers_map,
|
||||||
mock_get_nvme_controller.return_value = 'nvme1'
|
mock_run_nvme_cli):
|
||||||
|
mock_get_live_nvme_controllers_map.return_value = fake_controllers_map
|
||||||
mock_run_nvme_cli.side_effect = Exception()
|
mock_run_nvme_cli.side_effect = Exception()
|
||||||
self.assertRaises(exception.CommandExecutionFailed,
|
result = self.connector.rescan(EXECUTOR, TARGET_NQN)
|
||||||
self.connector.rescan, EXECUTOR, TARGET_NQN,
|
self.assertEqual(fake_controllers_map, result)
|
||||||
VOL_UUID)
|
mock_get_live_nvme_controllers_map.assert_called_with(EXECUTOR,
|
||||||
mock_get_nvme_controller.assert_called_with(EXECUTOR, TARGET_NQN)
|
TARGET_NQN)
|
||||||
nvme_command = ('ns-rescan', NVME_DEVICE_PATH)
|
nvme_command = ('ns-rescan', NVME_DEVICE_PATH)
|
||||||
mock_run_nvme_cli.assert_called_with(EXECUTOR, nvme_command)
|
mock_run_nvme_cli.assert_called_with(EXECUTOR, nvme_command)
|
||||||
|
|
||||||
|
@ -874,17 +891,17 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
|
|
||||||
@mock.patch.object(executor.Executor, '_execute')
|
@mock.patch.object(executor.Executor, '_execute')
|
||||||
@mock.patch.object(glob, 'glob')
|
@mock.patch.object(glob, 'glob')
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_controller')
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_nvme_controllers')
|
||||||
def test_get_nvme_device_path(self, mock_get_nvme_controller, mock_glob,
|
def test_get_nvme_device_path(self, mock_get_nvme_controllers, mock_glob,
|
||||||
mock_execute):
|
mock_execute):
|
||||||
mock_get_nvme_controller.return_value = 'nvme1'
|
mock_get_nvme_controllers.return_value = ['nvme1']
|
||||||
block_dev_path = '/sys/class/block/nvme1n*/uuid'
|
block_dev_path = '/sys/class/block/nvme1n*/uuid'
|
||||||
mock_glob.return_value = ['/sys/class/block/nvme1n1/uuid']
|
mock_glob.side_effect = [['/sys/class/block/nvme1n1/uuid']]
|
||||||
mock_execute.return_value = (VOL_UUID + "\n", "")
|
mock_execute.return_value = (VOL_UUID + "\n", "")
|
||||||
cmd = ['cat', '/sys/class/block/nvme1n1/uuid']
|
cmd = ['cat', '/sys/class/block/nvme1n1/uuid']
|
||||||
result = self.connector.get_nvme_device_path(EXECUTOR, TARGET_NQN,
|
result = self.connector.get_nvme_device_path(EXECUTOR, TARGET_NQN,
|
||||||
VOL_UUID)
|
VOL_UUID)
|
||||||
mock_get_nvme_controller.assert_called_with(EXECUTOR, TARGET_NQN)
|
mock_get_nvme_controllers.assert_called_with(EXECUTOR, TARGET_NQN)
|
||||||
self.assertEqual(NVME_NS_PATH, result)
|
self.assertEqual(NVME_NS_PATH, result)
|
||||||
mock_glob.assert_any_call(block_dev_path)
|
mock_glob.assert_any_call(block_dev_path)
|
||||||
args, kwargs = mock_execute.call_args
|
args, kwargs = mock_execute.call_args
|
||||||
|
@ -909,29 +926,25 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
if 'state' in value:
|
if 'state' in value:
|
||||||
return 'live' + "\n", ""
|
return 'live' + "\n", ""
|
||||||
|
|
||||||
@mock.patch.object(executor.Executor, '_execute',
|
@mock.patch.object(nvmeof.NVMeOFConnector, 'get_live_nvme_controllers_map')
|
||||||
side_effect=execute_side_effect)
|
def test_get_nvme_controllers(self, mock_get_live_nvme_controllers_map):
|
||||||
@mock.patch.object(glob, 'glob')
|
mock_get_live_nvme_controllers_map.return_value = fake_controllers_map
|
||||||
def test_get_nvme_controller(self, mock_glob, mock_execute):
|
result = self.connector.get_nvme_controllers(EXECUTOR, TARGET_NQN)
|
||||||
ctrl_path = '/sys/class/nvme-fabrics/ctl/nvme*'
|
fake_controllers_map_values = fake_controllers_map.values()
|
||||||
mock_glob.side_effect = [['/sys/class/nvme-fabrics/ctl/nvme1']]
|
self.assertEqual(list(fake_controllers_map_values)[0][1],
|
||||||
cmd = ['cat', '/sys/class/nvme-fabrics/ctl/nvme1/state']
|
list(result)[0][1])
|
||||||
result = self.connector._get_nvme_controller(EXECUTOR, TARGET_NQN)
|
mock_get_live_nvme_controllers_map.assert_called_with(EXECUTOR,
|
||||||
self.assertEqual('nvme1', result)
|
TARGET_NQN)
|
||||||
mock_glob.assert_any_call(ctrl_path)
|
|
||||||
args, kwargs = mock_execute.call_args
|
|
||||||
self.assertEqual(args[0], cmd[0])
|
|
||||||
self.assertEqual(args[1], cmd[1])
|
|
||||||
|
|
||||||
@mock.patch.object(executor.Executor, '_execute',
|
@mock.patch.object(executor.Executor, '_execute',
|
||||||
side_effect=execute_side_effect_not_live)
|
side_effect=execute_side_effect_not_live)
|
||||||
@mock.patch.object(glob, 'glob')
|
@mock.patch.object(glob, 'glob')
|
||||||
def test_get_nvme_controller_not_live(self, mock_glob, mock_execute):
|
def test_get_nvme_controllers_not_live(self, mock_glob, mock_execute):
|
||||||
ctrl_path = '/sys/class/nvme-fabrics/ctl/nvme*'
|
ctrl_path = '/sys/class/nvme-fabrics/ctl/nvme*'
|
||||||
mock_glob.side_effect = [['/sys/class/nvme-fabrics/ctl/nvme1']]
|
mock_glob.side_effect = [['/sys/class/nvme-fabrics/ctl/nvme1']]
|
||||||
cmd = ['cat', '/sys/class/nvme-fabrics/ctl/nvme1/state']
|
cmd = ['cat', '/sys/class/nvme-fabrics/ctl/nvme1/state']
|
||||||
self.assertRaises(exception.VolumeDeviceNotFound,
|
self.assertRaises(exception.VolumeDeviceNotFound,
|
||||||
self.connector._get_nvme_controller, EXECUTOR,
|
self.connector.get_nvme_controllers, EXECUTOR,
|
||||||
TARGET_NQN)
|
TARGET_NQN)
|
||||||
mock_glob.assert_any_call(ctrl_path)
|
mock_glob.assert_any_call(ctrl_path)
|
||||||
args, kwargs = mock_execute.call_args
|
args, kwargs = mock_execute.call_args
|
||||||
|
@ -941,12 +954,12 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
@mock.patch.object(executor.Executor, '_execute',
|
@mock.patch.object(executor.Executor, '_execute',
|
||||||
side_effect=execute_side_effect_not_found)
|
side_effect=execute_side_effect_not_found)
|
||||||
@mock.patch.object(glob, 'glob')
|
@mock.patch.object(glob, 'glob')
|
||||||
def test_get_nvme_controller_not_found(self, mock_glob, mock_execute):
|
def test_get_nvme_controllers_not_found(self, mock_glob, mock_execute):
|
||||||
ctrl_path = '/sys/class/nvme-fabrics/ctl/nvme*'
|
ctrl_path = '/sys/class/nvme-fabrics/ctl/nvme*'
|
||||||
mock_glob.side_effect = [['/sys/class/nvme-fabrics/ctl/nvme1']]
|
mock_glob.side_effect = [['/sys/class/nvme-fabrics/ctl/nvme1']]
|
||||||
cmd = ['cat', '/sys/class/nvme-fabrics/ctl/nvme1/state']
|
cmd = ['cat', '/sys/class/nvme-fabrics/ctl/nvme1/state']
|
||||||
self.assertRaises(exception.VolumeDeviceNotFound,
|
self.assertRaises(exception.VolumeDeviceNotFound,
|
||||||
self.connector._get_nvme_controller, EXECUTOR,
|
self.connector.get_nvme_controllers, EXECUTOR,
|
||||||
TARGET_NQN)
|
TARGET_NQN)
|
||||||
mock_glob.assert_any_call(ctrl_path)
|
mock_glob.assert_any_call(ctrl_path)
|
||||||
args, kwargs = mock_execute.call_args
|
args, kwargs = mock_execute.call_args
|
||||||
|
@ -1086,6 +1099,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
self.assertFalse(result)
|
self.assertFalse(result)
|
||||||
|
|
||||||
def _get_host_nqn(self):
|
def _get_host_nqn(self):
|
||||||
|
host_nqn = None
|
||||||
try:
|
try:
|
||||||
with open('/etc/nvme/hostnqn', 'r') as f:
|
with open('/etc/nvme/hostnqn', 'r') as f:
|
||||||
host_nqn = f.read().strip()
|
host_nqn = f.read().strip()
|
||||||
|
|
|
@ -42,6 +42,9 @@ class ZeroIntervalLoopingCall(loopingcall.FixedIntervalLoopingCall):
|
||||||
|
|
||||||
class ConnectorUtilsTestCase(test_base.TestCase):
|
class ConnectorUtilsTestCase(test_base.TestCase):
|
||||||
|
|
||||||
|
@mock.patch.object(nvmeof.NVMeOFConnector,
|
||||||
|
'_is_native_multipath_supported',
|
||||||
|
return_value=False)
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_system_uuid',
|
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_system_uuid',
|
||||||
return_value=None)
|
return_value=None)
|
||||||
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_host_uuid',
|
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_host_uuid',
|
||||||
|
@ -64,6 +67,7 @@ class ConnectorUtilsTestCase(test_base.TestCase):
|
||||||
mock_nqn,
|
mock_nqn,
|
||||||
mock_hostuuid,
|
mock_hostuuid,
|
||||||
mock_sysuuid,
|
mock_sysuuid,
|
||||||
|
mock_native_multipath_supported,
|
||||||
host='fakehost'):
|
host='fakehost'):
|
||||||
props_actual = connector.get_connector_properties('sudo',
|
props_actual = connector.get_connector_properties('sudo',
|
||||||
MY_IP,
|
MY_IP,
|
||||||
|
@ -76,6 +80,7 @@ class ConnectorUtilsTestCase(test_base.TestCase):
|
||||||
'host': host,
|
'host': host,
|
||||||
'ip': MY_IP,
|
'ip': MY_IP,
|
||||||
'multipath': multipath_result,
|
'multipath': multipath_result,
|
||||||
|
'nvme_native_multipath': False,
|
||||||
'os_type': os_type,
|
'os_type': os_type,
|
||||||
'platform': platform,
|
'platform': platform,
|
||||||
'do_local_attach': False}
|
'do_local_attach': False}
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Phase 1 (Native) implementation of NVMeoF Multipathing.
|
||||||
|
See cinder-specs/specs/yoga/nvme-multipath
|
Loading…
Reference in New Issue