Adding NVMEoF for initiator CLI
Added os-brick changes to support NVMEoF for initiator CLI. A New connector, NVMe connector, is added to handle initiator callsi for connecting and disconnecting volumes to instances using nvme-cli over RDMA. Implements: blueprint nvme-over-fabirc-nova Needed-By: I67a72c4226e54c18b3a6e4a13b5055fa6e85af09 Needed-By: I7cacd76c63e0ad29eb2d448ce07fbb5176f62721 Co-Authored-By: Ivan Kolodyazhny <e0ne@e0ne.info> Change-Id: I83697d6b46248edbe0a0ef3b76829b28ed5c048c
This commit is contained in:
parent
c1f1786307
commit
905e73129b
|
@ -61,3 +61,4 @@ SHEEPDOG = "SHEEPDOG"
|
||||||
VMDK = "VMDK"
|
VMDK = "VMDK"
|
||||||
GPFS = "GPFS"
|
GPFS = "GPFS"
|
||||||
VERITAS_HYPERSCALE = "VERITAS_HYPERSCALE"
|
VERITAS_HYPERSCALE = "VERITAS_HYPERSCALE"
|
||||||
|
NVME = "NVME"
|
||||||
|
|
|
@ -63,6 +63,7 @@ connector_list = [
|
||||||
'os_brick.initiator.windows.fibre_channel.WindowsFCConnector',
|
'os_brick.initiator.windows.fibre_channel.WindowsFCConnector',
|
||||||
'os_brick.initiator.windows.smbfs.WindowsSMBFSConnector',
|
'os_brick.initiator.windows.smbfs.WindowsSMBFSConnector',
|
||||||
'os_brick.initiator.connectors.vrtshyperscale.HyperScaleConnector',
|
'os_brick.initiator.connectors.vrtshyperscale.HyperScaleConnector',
|
||||||
|
'os_brick.initiator.connectors.nvme.NVMeConnector',
|
||||||
]
|
]
|
||||||
|
|
||||||
# Mappings used to determine who to contruct in the factory
|
# Mappings used to determine who to contruct in the factory
|
||||||
|
@ -110,6 +111,8 @@ _connector_mapping_linux = {
|
||||||
'os_brick.initiator.connectors.gpfs.GPFSConnector',
|
'os_brick.initiator.connectors.gpfs.GPFSConnector',
|
||||||
initiator.VERITAS_HYPERSCALE:
|
initiator.VERITAS_HYPERSCALE:
|
||||||
'os_brick.initiator.connectors.vrtshyperscale.HyperScaleConnector',
|
'os_brick.initiator.connectors.vrtshyperscale.HyperScaleConnector',
|
||||||
|
initiator.NVME:
|
||||||
|
'os_brick.initiator.connectors.nvme.NVMeConnector',
|
||||||
}
|
}
|
||||||
|
|
||||||
# Mapping for the S390X platform
|
# Mapping for the S390X platform
|
||||||
|
|
|
@ -0,0 +1,190 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
|
from oslo_concurrency import lockutils
|
||||||
|
from oslo_concurrency import processutils as putils
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from os_brick import exception
|
||||||
|
from os_brick import initiator
|
||||||
|
from os_brick.initiator.connectors import base
|
||||||
|
from os_brick import utils
|
||||||
|
|
||||||
|
|
||||||
|
DEVICE_SCAN_ATTEMPTS_DEFAULT = 3
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
synchronized = lockutils.synchronized_with_prefix('os-brick-')
|
||||||
|
|
||||||
|
|
||||||
|
class NVMeConnector(base.BaseLinuxConnector):
|
||||||
|
|
||||||
|
"""Connector class to attach/detach NVMe over fabric volumes."""
|
||||||
|
|
||||||
|
def __init__(self, root_helper, driver=None,
|
||||||
|
device_scan_attempts=initiator.DEVICE_SCAN_ATTEMPTS_DEFAULT,
|
||||||
|
*args, **kwargs):
|
||||||
|
super(NVMeConnector, self).__init__(
|
||||||
|
root_helper,
|
||||||
|
driver=driver,
|
||||||
|
device_scan_attempts=device_scan_attempts,
|
||||||
|
*args, **kwargs)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_connector_properties(root_helper, *args, **kwargs):
|
||||||
|
"""The NVMe connector properties."""
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def get_search_path(self):
|
||||||
|
return '/dev'
|
||||||
|
|
||||||
|
def get_volume_paths(self, connection_properties):
|
||||||
|
path = connection_properties['device_path']
|
||||||
|
LOG.debug("Path of volume to be extended is %(path)s", {'path': path})
|
||||||
|
return [path]
|
||||||
|
|
||||||
|
def _get_nvme_devices(self):
|
||||||
|
nvme_devices = []
|
||||||
|
pattern = r'/dev/nvme[0-9]n[0-9]'
|
||||||
|
cmd = ['nvme', 'list']
|
||||||
|
try:
|
||||||
|
(out, err) = self._execute(*cmd, root_helper=self._root_helper,
|
||||||
|
run_as_root=True)
|
||||||
|
for line in out.split('\n'):
|
||||||
|
result = re.match(pattern, line)
|
||||||
|
if result:
|
||||||
|
nvme_devices.append(result.group(0))
|
||||||
|
LOG.debug("_get_nvme_devices returned %(nvme_devices)s",
|
||||||
|
{'nvme_devices': nvme_devices})
|
||||||
|
return nvme_devices
|
||||||
|
|
||||||
|
except putils.ProcessExecutionError:
|
||||||
|
LOG.error(
|
||||||
|
"Failed to list available NVMe connected controllers.")
|
||||||
|
raise
|
||||||
|
|
||||||
|
@utils.trace
|
||||||
|
@synchronized('connect_volume')
|
||||||
|
def connect_volume(self, connection_properties):
|
||||||
|
"""Discover and attach the volume.
|
||||||
|
|
||||||
|
:param connection_properties: The dictionary that describes all
|
||||||
|
of the target volume attributes.
|
||||||
|
connection_properties must include:
|
||||||
|
nqn - NVMe subsystem name to the volume to be connected
|
||||||
|
target_port - NVMe target port that hosts the nqn sybsystem
|
||||||
|
target_portal - NVMe target ip that hosts the nqn sybsystem
|
||||||
|
:type connection_properties: dict
|
||||||
|
:returns: dict
|
||||||
|
"""
|
||||||
|
|
||||||
|
current_nvme_devices = self._get_nvme_devices()
|
||||||
|
|
||||||
|
device_info = {'type': 'block'}
|
||||||
|
conn_nqn = connection_properties['nqn'].split('.', 1)[1]
|
||||||
|
target_portal = connection_properties['target_portal']
|
||||||
|
port = connection_properties['target_port']
|
||||||
|
nvme_transport_type = connection_properties['transport_type']
|
||||||
|
cmd = [
|
||||||
|
'nvme', 'connect',
|
||||||
|
'-t', nvme_transport_type,
|
||||||
|
'-n', conn_nqn,
|
||||||
|
'-a', target_portal,
|
||||||
|
'-s', port]
|
||||||
|
try:
|
||||||
|
self._execute(*cmd, root_helper=self._root_helper,
|
||||||
|
run_as_root=True)
|
||||||
|
except putils.ProcessExecutionError:
|
||||||
|
LOG.error(
|
||||||
|
"Failed to connect to NVMe nqn "
|
||||||
|
"%(conn_nqn)s", {'conn_nqn': conn_nqn})
|
||||||
|
raise
|
||||||
|
|
||||||
|
for retry in range(1, self.device_scan_attempts + 1):
|
||||||
|
all_nvme_devices = self._get_nvme_devices()
|
||||||
|
path = set(all_nvme_devices) - set(current_nvme_devices)
|
||||||
|
if path:
|
||||||
|
break
|
||||||
|
time.sleep(retry ** 2)
|
||||||
|
else:
|
||||||
|
LOG.error("Failed to retrieve available connected NVMe "
|
||||||
|
"controllers when running nvme list")
|
||||||
|
raise exception.TargetPortalNotFound(target_portal)
|
||||||
|
|
||||||
|
path = list(path)
|
||||||
|
LOG.debug("all_nvme_devices are %(all_nvme_devices)s",
|
||||||
|
{'all_nvme_devices': all_nvme_devices})
|
||||||
|
device_info['path'] = path[0]
|
||||||
|
LOG.debug("NVMe device to be connected to is %(path)s",
|
||||||
|
{'path': path[0]})
|
||||||
|
return device_info
|
||||||
|
|
||||||
|
@utils.trace
|
||||||
|
@synchronized('disconnect_volume')
|
||||||
|
def disconnect_volume(self, connection_properties, device_info):
|
||||||
|
"""Detach and flush the volume.
|
||||||
|
|
||||||
|
:param connection_properties: The dictionary that describes all
|
||||||
|
of the target volume attributes.
|
||||||
|
connection_properties must include:
|
||||||
|
device_path - path to the volume to be connected
|
||||||
|
:type connection_properties: dict
|
||||||
|
|
||||||
|
:param device_info: historical difference, but same as connection_props
|
||||||
|
:type device_info: dict
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
conn_nqn = connection_properties['nqn']
|
||||||
|
device_path = connection_properties['device_path']
|
||||||
|
LOG.debug(
|
||||||
|
"Trying to disconnect from NVMe nqn "
|
||||||
|
"%(conn_nqn)s with device_path %(device_path)s",
|
||||||
|
{'conn_nqn': conn_nqn, 'device_path': device_path})
|
||||||
|
cmd = [
|
||||||
|
'nvme',
|
||||||
|
'disconnect',
|
||||||
|
'-d',
|
||||||
|
device_path]
|
||||||
|
try:
|
||||||
|
self._execute(
|
||||||
|
*cmd,
|
||||||
|
root_helper=self._root_helper,
|
||||||
|
run_as_root=True)
|
||||||
|
|
||||||
|
except putils.ProcessExecutionError:
|
||||||
|
LOG.error(
|
||||||
|
"Failed to disconnect from NVMe nqn "
|
||||||
|
"%(conn_nqn)s with device_path %(device_path)s",
|
||||||
|
{'conn_nqn': conn_nqn, 'device_path': device_path})
|
||||||
|
raise
|
||||||
|
|
||||||
|
@utils.trace
|
||||||
|
@synchronized('extend_volume')
|
||||||
|
def extend_volume(self, connection_properties):
|
||||||
|
"""Update the local kernel's size information.
|
||||||
|
|
||||||
|
Try and update the local kernel's size information
|
||||||
|
for an LVM volume.
|
||||||
|
"""
|
||||||
|
volume_paths = self.get_volume_paths(connection_properties)
|
||||||
|
if volume_paths:
|
||||||
|
return self._linuxscsi.extend_volume(volume_paths)
|
||||||
|
else:
|
||||||
|
LOG.warning("Couldn't find any volume paths on the host to "
|
||||||
|
"extend volume for %(props)s",
|
||||||
|
{'props': connection_properties})
|
||||||
|
raise exception.VolumePathsNotFound()
|
|
@ -0,0 +1,168 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from oslo_concurrency import processutils as putils
|
||||||
|
|
||||||
|
from os_brick import exception
|
||||||
|
from os_brick.initiator.connectors import nvme
|
||||||
|
from os_brick.initiator import linuxscsi
|
||||||
|
from os_brick.tests.initiator import test_connector
|
||||||
|
|
||||||
|
FAKE_NVME_LIST_OUTPUT = """
|
||||||
|
Node SN Model \
|
||||||
|
Namespace Usage Format FW Rev\n
|
||||||
|
---------------- -------------------- ---------------------------------------\
|
||||||
|
- --------- -------------------------- ---------------- --------\n
|
||||||
|
/dev/nvme0n1 67ff9467da6e5567 Linux \
|
||||||
|
10 1.07 GB / 1.07 GB 512 B + 0 B 4.8.0-58\n
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class NVMeConnectorTestCase(test_connector.ConnectorTestCase):
|
||||||
|
|
||||||
|
"""Test cases for NVMe initiator class."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(NVMeConnectorTestCase, self).setUp()
|
||||||
|
self.connector = nvme.NVMeConnector(None,
|
||||||
|
execute=self.fake_execute)
|
||||||
|
|
||||||
|
def _nvme_list_cmd(self, *args, **kwargs):
|
||||||
|
return FAKE_NVME_LIST_OUTPUT, None
|
||||||
|
|
||||||
|
def test__get_nvme_devices(self):
|
||||||
|
expected = ['/dev/nvme0n1']
|
||||||
|
self.connector._execute = self._nvme_list_cmd
|
||||||
|
actual = self.connector._get_nvme_devices()
|
||||||
|
self.assertEqual(expected, actual)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_execute')
|
||||||
|
def test_get_nvme_devices_raise(self, mock_execute):
|
||||||
|
mock_execute.side_effect = putils.ProcessExecutionError
|
||||||
|
self.assertRaises(putils.ProcessExecutionError,
|
||||||
|
self.connector._get_nvme_devices)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_get_nvme_devices')
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_execute')
|
||||||
|
@mock.patch('time.sleep')
|
||||||
|
def test_connect_volume(self, mock_sleep, mock_execute, mock_devices):
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
|
||||||
|
mock_devices.side_effect = [
|
||||||
|
['/dev/nvme0n1'], ['/dev/nvme0n2']]
|
||||||
|
|
||||||
|
device_info = self.connector.connect_volume(
|
||||||
|
connection_properties)
|
||||||
|
self.assertEqual('/dev/nvme0n2', device_info['path'])
|
||||||
|
self.assertEqual('block', device_info['type'])
|
||||||
|
|
||||||
|
self.assertEqual(2, mock_devices.call_count)
|
||||||
|
|
||||||
|
mock_execute.assert_called_once_with(
|
||||||
|
'nvme', 'connect', '-t',
|
||||||
|
connection_properties['transport_type'], '-n',
|
||||||
|
'volume_123',
|
||||||
|
'-a', connection_properties['target_portal'],
|
||||||
|
'-s', connection_properties['target_port'],
|
||||||
|
root_helper=None,
|
||||||
|
run_as_root=True)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_execute')
|
||||||
|
def test_connect_volume_raise(self, mock_execute):
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
mock_execute.side_effect = putils.ProcessExecutionError
|
||||||
|
self.assertRaises(putils.ProcessExecutionError,
|
||||||
|
self.connector.connect_volume,
|
||||||
|
connection_properties)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_get_nvme_devices')
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_execute')
|
||||||
|
@mock.patch('time.sleep')
|
||||||
|
def test_connect_volume_max_retry(
|
||||||
|
self, mock_sleep, mock_execute, mock_devices):
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
|
||||||
|
mock_devices.return_value = '/dev/nvme0n1'
|
||||||
|
|
||||||
|
self.assertRaises(exception.TargetPortalNotFound,
|
||||||
|
self.connector.connect_volume,
|
||||||
|
connection_properties)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_execute')
|
||||||
|
def test_disconnect_volume(self, mock_execute):
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
self.connector.disconnect_volume(connection_properties, None)
|
||||||
|
|
||||||
|
mock_execute.asert_called_once_with(
|
||||||
|
'nvme', 'disconnect', '-n',
|
||||||
|
'volume_123',
|
||||||
|
root_helper=None,
|
||||||
|
run_as_root=True)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, '_execute')
|
||||||
|
def test_disconnect_volume_raise(self, mock_execute):
|
||||||
|
mock_execute.side_effect = putils.ProcessExecutionError
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
|
||||||
|
self.assertRaises(putils.ProcessExecutionError,
|
||||||
|
self.connector.disconnect_volume,
|
||||||
|
connection_properties,
|
||||||
|
None)
|
||||||
|
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, 'get_volume_paths')
|
||||||
|
def test_extend_volume_no_path(self, mock_volume_paths):
|
||||||
|
mock_volume_paths.return_value = []
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
|
||||||
|
self.assertRaises(exception.VolumePathsNotFound,
|
||||||
|
self.connector.extend_volume,
|
||||||
|
connection_properties)
|
||||||
|
|
||||||
|
@mock.patch.object(linuxscsi.LinuxSCSI, 'extend_volume')
|
||||||
|
@mock.patch.object(nvme.NVMeConnector, 'get_volume_paths')
|
||||||
|
def test_extend_volume(self, mock_volume_paths, mock_scsi_extend):
|
||||||
|
fake_new_size = 1024
|
||||||
|
mock_volume_paths.return_value = ['/dev/vdx']
|
||||||
|
mock_scsi_extend.return_value = fake_new_size
|
||||||
|
connection_properties = {'target_portal': 'portal',
|
||||||
|
'target_port': 1,
|
||||||
|
'nqn': 'nqn.volume_123',
|
||||||
|
'device_path': '',
|
||||||
|
'transport_type': 'rdma'}
|
||||||
|
new_size = self.connector.extend_volume(connection_properties)
|
||||||
|
self.assertEqual(fake_new_size, new_size)
|
Loading…
Reference in New Issue