diff --git a/os_brick/initiator/connectors/nvmeof.py b/os_brick/initiator/connectors/nvmeof.py index cf7d0b1d5..c71b87584 100644 --- a/os_brick/initiator/connectors/nvmeof.py +++ b/os_brick/initiator/connectors/nvmeof.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import json import re import time @@ -126,6 +127,99 @@ class NVMeOFConnector(base.BaseLinuxConnector): self._execute(*cmd, root_helper=self._root_helper, run_as_root=True) + def _get_nvme_subsys(self): + # Example output: + # { + # 'Subsystems' : [ + # { + # 'Name' : 'nvme-subsys0', + # 'NQN' : 'nqn.2016-06.io.spdk:cnode1' + # }, + # { + # 'Paths' : [ + # { + # 'Name' : 'nvme0', + # 'Transport' : 'rdma', + # 'Address' : 'traddr=10.0.2.15 trsvcid=4420' + # } + # ] + # } + # ] + # } + # + + cmd = ['nvme', 'list-subsys', '-o', 'json'] + ret_val = self._execute(*cmd, root_helper=self._root_helper, + run_as_root=True) + + return ret_val + + @utils.retry(exceptions=exception.NotFound, retries=5) + def _is_nvme_available(self, nvme_name): + nvme_name_pattern = "/dev/%sn[0-9]+" % nvme_name + for nvme_dev_name in self._get_nvme_devices(): + if re.match(nvme_name_pattern, nvme_dev_name): + return True + else: + LOG.error("Failed to find nvme device") + raise exception.NotFound() + + def _wait_for_blk(self, nvme_transport_type, conn_nqn, + target_portal, port): + # Find nvme name in subsystem list and wait max 15 seconds + # until new volume will be available in kernel + nvme_name = "" + nvme_address = "traddr=%s trsvcid=%s" % (target_portal, port) + + # Get nvme subsystems in order to find + # nvme name for connected nvme + try: + (out, err) = self._get_nvme_subsys() + except putils.ProcessExecutionError: + LOG.error("Failed to get nvme subsystems") + raise + + # Get subsystem list. Throw exception if out is currupt or empty + try: + subsystems = json.loads(out)['Subsystems'] + except Exception: + return False + + # Find nvme name among subsystems + for i in range(0, int(len(subsystems) / 2)): + subsystem = subsystems[i * 2] + if 'NQN' in subsystem and subsystem['NQN'] == conn_nqn: + for path in subsystems[i * 2 + 1]['Paths']: + if (path['Transport'] == nvme_transport_type + and path['Address'] == nvme_address): + nvme_name = path['Name'] + break + + if not nvme_name: + return False + + # Wait until nvme will be available in kernel + return self._is_nvme_available(nvme_name) + + def _try_disconnect_volume(self, conn_nqn, ignore_errors=False): + cmd = [ + 'nvme', + 'disconnect', + '-n', + conn_nqn] + try: + self._execute( + *cmd, + root_helper=self._root_helper, + run_as_root=True) + + except putils.ProcessExecutionError: + LOG.error( + "Failed to disconnect from NVMe nqn " + "%(conn_nqn)s", {'conn_nqn': conn_nqn}) + if not ignore_errors: + raise + @utils.trace @synchronized('connect_volume') def connect_volume(self, connection_properties): @@ -159,7 +253,14 @@ class NVMeOFConnector(base.BaseLinuxConnector): cmd.extend(['-q', host_nqn]) self._try_connect_nvme(cmd) - + try: + self._wait_for_blk(nvme_transport_type, conn_nqn, + target_portal, port) + except exception.NotFound: + LOG.error("Waiting for nvme failed") + self._try_disconnect_volume(conn_nqn, True) + raise exception.NotFound(message="nvme connect: NVMe device " + "not found") path = self._get_device_path(current_nvme_devices) device_info['path'] = path[0] LOG.debug("NVMe device to be connected to is %(path)s", diff --git a/os_brick/tests/initiator/connectors/test_nvmeof.py b/os_brick/tests/initiator/connectors/test_nvmeof.py index 5e423de31..6a502753e 100644 --- a/os_brick/tests/initiator/connectors/test_nvmeof.py +++ b/os_brick/tests/initiator/connectors/test_nvmeof.py @@ -12,6 +12,7 @@ import mock +import ddt from oslo_concurrency import processutils as putils from os_brick import exception @@ -30,7 +31,51 @@ Node SN Model \ 1 3.22 GB / 3.22 GB 512 B + 0 B 4.8.0-56\n """ +FAKE_NVME_LIST_SUBSYS = """ +{ + "Subsystems" : [ + { + "Name" : "nvme-subsys0", + "NQN" : "nqn.fake:cnode1" + }, + { + "Paths" : [ + { + "Name" : "nvme0", + "Transport" : "rdma", + "Address" : "traddr=10.0.2.15 trsvcid=4420" + } + ] + }, + { + "Name" : "nvme-subsys1", + "NQN" : "nqn.2016-06.io.spdk:cnode1" + }, + { + "Paths" : [ + { + "Name" : "nvme1", + "Transport" : "rdma", + "Address" : "traddr=10.0.2.15 trsvcid=4420" + } + ] + } + ] +} +""" +NVME_DATA1 = {'nvme_transport_type': 'rdma', + 'conn_nqn': 'nqn.2016-06.io.spdk:cnode1', + 'target_portal': '10.0.2.15', + 'port': '4420'} + +NVME_DATA2 = {'nvme_transport_type': 'rdma', + 'conn_nqn': 'nqn.2016-06.io.spdk:cnode2', + 'target_portal': '10.0.2.15', + 'port': '4420'} + + +@ddt.ddt class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): """Test cases for NVMe initiator class.""" @@ -76,6 +121,64 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): actual = self.connector._get_nvme_devices() self.assertEqual(expected, actual) + @ddt.unpack + @ddt.data({'expected': True, 'nvme': NVME_DATA1, + 'list_subsys': FAKE_NVME_LIST_SUBSYS, + 'nvme_list': ['/dev/nvme0n1', '/dev/nvme1n1']}, + {'expected': False, 'nvme': NVME_DATA2, + 'list_subsys': FAKE_NVME_LIST_SUBSYS, + 'nvme_list': ['/dev/nvme1n1']}, + {'expected': False, 'nvme': NVME_DATA1, + 'list_subsys': '{}', + 'nvme_list': ['dev/nvme1n1']}) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', + autospec=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys', + autospec=True) + @mock.patch('time.sleep', autospec=True) + def test__wait_for_blk(self, mock_sleep, mock_nvme_subsys, + mock_nvme_dev, expected, nvme, + list_subsys, nvme_list): + mock_nvme_subsys.return_value = (list_subsys, "") + mock_nvme_dev.return_value = nvme_list + actual = self.connector._wait_for_blk(**nvme) + self.assertEqual(expected, actual) + + @ddt.unpack + @ddt.data({'expected': False, 'nvme': NVME_DATA1, + 'list_subsys': FAKE_NVME_LIST_SUBSYS, + 'nvme_list': ['/dev/nvme0n1']}) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', + autospec=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys', + autospec=True) + @mock.patch('time.sleep', autospec=True) + def test__wait_for_blk_raise(self, mock_sleep, mock_nvme_subsys, + mock_nvme_dev, expected, nvme, + list_subsys, nvme_list): + mock_nvme_subsys.return_value = (list_subsys, "") + mock_nvme_dev.return_value = nvme_list + self.assertRaises(exception.NotFound, + self.connector._wait_for_blk, + **nvme) + + @ddt.unpack + @ddt.data({'expected': True, 'nvme': NVME_DATA1, + 'list_subsys': FAKE_NVME_LIST_SUBSYS, + 'nvme_list': ['dev/nvme0n1', '/dev/nvme1n1']}) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', + autospec=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys', + autospec=True) + @mock.patch('time.sleep', autospec=True) + def test__wait_for_blk_retry_success(self, mock_sleep, mock_nvme_subsys, + mock_nvme_dev, expected, nvme, + list_subsys, nvme_list): + mock_nvme_subsys.return_value = (list_subsys, "") + mock_nvme_dev.side_effect = [[], nvme_list] + actual = self.connector._wait_for_blk(**nvme) + self.assertEqual(expected, actual) + @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) @mock.patch('time.sleep', autospec=True) def test_get_nvme_devices_raise(self, mock_sleep, mock_execute): @@ -83,11 +186,14 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): self.assertRaises(exception.CommandExecutionFailed, self.connector._get_nvme_devices) + @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk', + autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) @mock.patch('time.sleep', autospec=True) - def test_connect_volume(self, mock_sleep, mock_execute, mock_devices): + def test_connect_volume(self, mock_sleep, mock_execute, mock_devices, + mock_blk): connection_properties = {'target_portal': 'portal', 'target_port': 1, 'nqn': 'nqn.volume_123', @@ -96,6 +202,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): mock_devices.side_effect = [ ['/dev/nvme0n1'], ['/dev/nvme0n2']] + mock_blk.return_value = True device_info = self.connector.connect_volume( connection_properties) @@ -114,12 +221,15 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): root_helper=None, run_as_root=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk', + autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) @mock.patch('time.sleep', autospec=True) def test_connect_volume_hostnqn( - self, mock_sleep, mock_execute, mock_devices): + self, mock_sleep, mock_execute, mock_devices, + mock_blk): connection_properties = {'target_portal': 'portal', 'target_port': 1, 'nqn': 'nqn.volume_123', @@ -129,6 +239,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): mock_devices.side_effect = [ ['/dev/nvme0n1'], ['/dev/nvme0n2']] + mock_blk.return_value = True device_info = self.connector.connect_volume( connection_properties) @@ -161,12 +272,36 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): self.connector.connect_volume, connection_properties) + @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', + autospec=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_subsys', + autospec=True) + @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk', + autospec=True) + @mock.patch('time.sleep', autospec=True) + def test_connect_volume_wait_for_blk_raise(self, mock_sleep, mock_blk, + mock_subsys, mock_devices, + mock_execute): + connection_properties = {'target_portal': 'portal', + 'target_port': 1, + 'nqn': 'nqn.volume_123', + 'device_path': '', + 'transport_type': 'rdma'} + mock_blk.side_effect = exception.NotFound + self.assertRaises(exception.NotFound, + self.connector.connect_volume, + connection_properties) + + @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk', + autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) @mock.patch('time.sleep', autospec=True) def test_connect_volume_max_retry( - self, mock_sleep, mock_execute, mock_devices): + self, mock_sleep, mock_execute, mock_devices, + mock_blk): connection_properties = {'target_portal': 'portal', 'target_port': 1, 'nqn': 'nqn.volume_123', @@ -174,17 +309,21 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): 'transport_type': 'rdma'} mock_devices.return_value = '/dev/nvme0n1' + mock_blk.return_value = True self.assertRaises(exception.VolumePathsNotFound, self.connector.connect_volume, connection_properties) + @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk', + autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) @mock.patch('time.sleep', autospec=True) def test_connect_volume_nvmelist_retry_success( - self, mock_sleep, mock_execute, mock_devices): + self, mock_sleep, mock_execute, mock_devices, + mock_blk): connection_properties = {'target_portal': 'portal', 'target_port': 1, 'nqn': 'nqn.volume_123', @@ -194,17 +333,21 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): ['/dev/nvme0n1'], ['/dev/nvme0n1'], ['/dev/nvme0n1', '/dev/nvme0n2']] + mock_blk.return_value = True device_info = self.connector.connect_volume( connection_properties) self.assertEqual('/dev/nvme0n2', device_info['path']) self.assertEqual('block', device_info['type']) + @mock.patch.object(nvmeof.NVMeOFConnector, '_wait_for_blk', + autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_get_nvme_devices', autospec=True) @mock.patch.object(nvmeof.NVMeOFConnector, '_execute', autospec=True) @mock.patch('time.sleep', autospec=True) - def test_connect_nvmeof_retry_success( - self, mock_sleep, mock_execute, mock_devices): + def test_connect_nvme_retry_success( + self, mock_sleep, mock_execute, mock_devices, + mock_blk): connection_properties = {'target_portal': 'portal', 'target_port': 1, 'nqn': 'nqn.volume_123', @@ -213,6 +356,7 @@ class NVMeOFConnectorTestCase(test_connector.ConnectorTestCase): mock_devices.side_effect = [ ['/dev/nvme0n1'], ['/dev/nvme0n1', '/dev/nvme0n2']] + mock_blk.return_value = True device_info = self.connector.connect_volume( connection_properties) mock_execute.side_effect = [