openstack
/
os-brick
Code Issues Proposed changes

Merge "connectors/nvme: Wait until nvme device shows up in kernel"

This commit is contained in:
Zuul 2020-03-12 11:34:26 +00:00 committed by Gerrit Code Review
parent 3be101e628 5d7517572d
commit 9649f17228
2 changed files with 252 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
os_brick/initiator/connectors/nvmeof.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import re
import time
@ -126,6 +127,99 @@ class NVMeOFConnector(base.BaseLinuxConnector):
self._execute(*cmd, root_helper=self._root_helper,
run_as_root=True)
def _get_nvme_subsys(self):
# Example output:
# {
# 'Subsystems' : [
# {
# 'Name' : 'nvme-subsys0',
# 'NQN' : 'nqn.2016-06.io.spdk:cnode1'
# },
# {
# 'Paths' : [
# {
# 'Name' : 'nvme0',
# 'Transport' : 'rdma',
# 'Address' : 'traddr=10.0.2.15 trsvcid=4420'
# }
# ]
# }
# ]
# }
#
cmd = ['nvme', 'list-subsys', '-o', 'json']
ret_val = self._execute(*cmd, root_helper=self._root_helper,
run_as_root=True)
return ret_val
@utils.retry(exceptions=exception.NotFound, retries=5)
def _is_nvme_available(self, nvme_name):
nvme_name_pattern = "/dev/%sn[0-9]+" % nvme_name
for nvme_dev_name in self._get_nvme_devices():
if re.match(nvme_name_pattern, nvme_dev_name):
return True
else:
LOG.error("Failed to find nvme device")
raise exception.NotFound()
def _wait_for_blk(self, nvme_transport_type, conn_nqn,
target_portal, port):
# Find nvme name in subsystem list and wait max 15 seconds
# until new volume will be available in kernel
nvme_name = ""
nvme_address = "traddr=%s trsvcid=%s" % (target_portal, port)
# Get nvme subsystems in order to find
# nvme name for connected nvme
try:
(out, err) = self._get_nvme_subsys()
except putils.ProcessExecutionError:
LOG.error("Failed to get nvme subsystems")
raise
# Get subsystem list. Throw exception if out is currupt or empty
try:
subsystems = json.loads(out)['Subsystems']
except Exception:
return False
# Find nvme name among subsystems
for i in range(0, int(len(subsystems) / 2)):
subsystem = subsystems[i * 2]
if 'NQN' in subsystem and subsystem['NQN'] == conn_nqn:
for path in subsystems[i * 2 + 1]['Paths']:
if (path['Transport'] == nvme_transport_type
and path['Address'] == nvme_address):
nvme_name = path['Name']
break
if not nvme_name:
return False
# Wait until nvme will be available in kernel
return self._is_nvme_available(nvme_name)
def _try_disconnect_volume(self, conn_nqn, ignore_errors=False):
cmd = [
'nvme',
'disconnect',
'-n',
conn_nqn]
try:
self._execute(
*cmd,
root_helper=self._root_helper,
run_as_root=True)
except putils.ProcessExecutionError:
LOG.error(
"Failed to disconnect from NVMe nqn "
"%(conn_nqn)s", {'conn_nqn': conn_nqn})
if not ignore_errors:
raise
@utils.trace
@synchronized('connect_volume')
def connect_volume(self, connection_properties):
@ -159,7 +253,14 @@ class NVMeOFConnector(base.BaseLinuxConnector):
cmd.extend(['-q', host_nqn])
self._try_connect_nvme(cmd)
try:
self._wait_for_blk(nvme_transport_type, conn_nqn,
target_portal, port)
except exception.NotFound:
LOG.error("Waiting for nvme failed")
self._try_disconnect_volume(conn_nqn, True)
raise exception.NotFound(message="nvme connect: NVMe device "
"not found")
path = self._get_device_path(current_nvme_devices)
device_info['path'] = path[0]
LOG.debug("NVMe device to be connected to is %(path)s",

156
os_brick/tests/initiator/connectors/test_nvmeof.py
View File

@ -12,6 +12,7 @@
import mock
import ddt
from oslo_concurrency import processutils as putils
from os_brick import exception
@ -30,7 +31,51 @@ Node SN Model \
1 3.22 GB / 3.22 GB 512 B + 0 B 4.8.0-56\n
"""
FAKE_NVME_LIST_SUBSYS = """
{
"Subsystems" : [
{
"Name" : "nvme-subsys0",
"NQN" : "nqn.fake:cnode1"
},
{
"Paths" : [
{
"Name" : "nvme0",
"Transport" : "rdma",
"Address" : "traddr=10.0.2.15 trsvcid=4420"
}
]
},
{
"Name" : "nvme-subsys1",
"NQN" : "nqn.2016-06.io.spdk:cnode1"
},
{
"Paths" : [
{
"Name" : "nvme1",
"Transport" : "rdma",
"Address" : "traddr=10.0.2.15 trsvcid=4420"
}
]
}
]
}
"""
NVME_DATA1 = {'nvme_transport_type': 'rdma',
'conn_nqn': 'nqn.2016-06.io.spdk:cnode1',
'target_portal': '10.0.2.15',
'port': '4420'}
NVME_DATA2 = {'nvme_transport_type': 'rdma',
'conn_nqn': 'nqn.2016-06.io.spdk:cnode2',
'target_portal': '10.0.2.15',
'port': '4420'}
@ddt.ddt
class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
"""Test cases for NVMe initiator class."""
@ -76,6 +121,64 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
actual = self.connector._get_nvme_devices()
self.assertEqual(expected, actual)
@ddt.unpack
@ddt.data({'expected': True, 'nvme': NVME_DATA1,
'list_subsys': FAKE_NVME_LIST_SUBSYS,
'nvme_list': ['/dev/nvme0n1', '/dev/nvme1n1']},
{'expected': False, 'nvme': NVME_DATA2,
'list_subsys': FAKE_NVME_LIST_SUBSYS,
'nvme_list': ['/dev/nvme1n1']},
{'expected': False, 'nvme': NVME_DATA1,
'list_subsys': '{}',
'nvme_list': ['dev/nvme1n1']})
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
autospec=True)
@mock.patch('time.sleep', autospec=True)
def test__wait_for_blk(self, mock_sleep, mock_nvme_subsys,
mock_nvme_dev, expected, nvme,
list_subsys, nvme_list):
mock_nvme_subsys.return_value = (list_subsys, "")
mock_nvme_dev.return_value = nvme_list
actual = self.connector._wait_for_blk(**nvme)
self.assertEqual(expected, actual)
@ddt.unpack
@ddt.data({'expected': False, 'nvme': NVME_DATA1,
'list_subsys': FAKE_NVME_LIST_SUBSYS,
'nvme_list': ['/dev/nvme0n1']})
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
autospec=True)
@mock.patch('time.sleep', autospec=True)
def test__wait_for_blk_raise(self, mock_sleep, mock_nvme_subsys,
mock_nvme_dev, expected, nvme,
list_subsys, nvme_list):
mock_nvme_subsys.return_value = (list_subsys, "")
mock_nvme_dev.return_value = nvme_list
self.assertRaises(exception.NotFound,
self.connector._wait_for_blk,
**nvme)
@ddt.unpack
@ddt.data({'expected': True, 'nvme': NVME_DATA1,
'list_subsys': FAKE_NVME_LIST_SUBSYS,
'nvme_list': ['dev/nvme0n1', '/dev/nvme1n1']})
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
autospec=True)
@mock.patch('time.sleep', autospec=True)
def test__wait_for_blk_retry_success(self, mock_sleep, mock_nvme_subsys,
mock_nvme_dev, expected, nvme,
list_subsys, nvme_list):
mock_nvme_subsys.return_value = (list_subsys, "")
mock_nvme_dev.side_effect = [[], nvme_list]
actual = self.connector._wait_for_blk(**nvme)
self.assertEqual(expected, actual)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_get_nvme_devices_raise(self, mock_sleep, mock_execute):
@ -83,11 +186,14 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
self.assertRaises(exception.CommandExecutionFailed,
self.connector._get_nvme_devices)
@mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_connect_volume(self, mock_sleep, mock_execute, mock_devices):
def test_connect_volume(self, mock_sleep, mock_execute, mock_devices,
mock_blk):
connection_properties = {'target_portal': 'portal',
'target_port': 1,
'nqn': 'nqn.volume_123',
@ -96,6 +202,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
mock_devices.side_effect = [
['/dev/nvme0n1'], ['/dev/nvme0n2']]
mock_blk.return_value = True
device_info = self.connector.connect_volume(
connection_properties)
@ -114,12 +221,15 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
root_helper=None,
run_as_root=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_connect_volume_hostnqn(
self, mock_sleep, mock_execute, mock_devices):
self, mock_sleep, mock_execute, mock_devices,
mock_blk):
connection_properties = {'target_portal': 'portal',
'target_port': 1,
'nqn': 'nqn.volume_123',
@ -129,6 +239,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
mock_devices.side_effect = [
['/dev/nvme0n1'], ['/dev/nvme0n2']]
mock_blk.return_value = True
device_info = self.connector.connect_volume(
connection_properties)
@ -161,12 +272,36 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
self.connector.connect_volume,
connection_properties)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_connect_volume_wait_for_blk_raise(self, mock_sleep, mock_blk,
mock_subsys, mock_devices,
mock_execute):
connection_properties = {'target_portal': 'portal',
'target_port': 1,
'nqn': 'nqn.volume_123',
'device_path': '',
'transport_type': 'rdma'}
mock_blk.side_effect = exception.NotFound
self.assertRaises(exception.NotFound,
self.connector.connect_volume,
connection_properties)
@mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_connect_volume_max_retry(
self, mock_sleep, mock_execute, mock_devices):
self, mock_sleep, mock_execute, mock_devices,
mock_blk):
connection_properties = {'target_portal': 'portal',
'target_port': 1,
'nqn': 'nqn.volume_123',
@ -174,17 +309,21 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
'transport_type': 'rdma'}
mock_devices.return_value = '/dev/nvme0n1'
mock_blk.return_value = True
self.assertRaises(exception.VolumePathsNotFound,
self.connector.connect_volume,
connection_properties)
@mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_connect_volume_nvmelist_retry_success(
self, mock_sleep, mock_execute, mock_devices):
self, mock_sleep, mock_execute, mock_devices,
mock_blk):
connection_properties = {'target_portal': 'portal',
'target_port': 1,
'nqn': 'nqn.volume_123',
@ -194,17 +333,21 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
['/dev/nvme0n1'],
['/dev/nvme0n1'],
['/dev/nvme0n1', '/dev/nvme0n2']]
mock_blk.return_value = True
device_info = self.connector.connect_volume(
connection_properties)
self.assertEqual('/dev/nvme0n2', device_info['path'])
self.assertEqual('block', device_info['type'])
@mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices',
autospec=True)
@mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True)
@mock.patch('time.sleep', autospec=True)
def test_connect_nvmeof_retry_success(
self, mock_sleep, mock_execute, mock_devices):
def test_connect_nvme_retry_success(
self, mock_sleep, mock_execute, mock_devices,
mock_blk):
connection_properties = {'target_portal': 'portal',
'target_port': 1,
'nqn': 'nqn.volume_123',
@ -213,6 +356,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase):
mock_devices.side_effect = [
['/dev/nvme0n1'],
['/dev/nvme0n1', '/dev/nvme0n2']]
mock_blk.return_value = True
device_info = self.connector.connect_volume(
connection_properties)
mock_execute.side_effect = [