Add NVMe/TCP support to Dell EMC PowerStore driver

Implements: blueprint powerstore-nvme-tcp
Change-Id: If92dd2588edccdac977e738f3497f5f2bac3f146
This commit is contained in:
Jean-Pierre Roquesalane 2021-11-24 17:46:46 +01:00 committed by olegnest
parent 311465e501
commit dd2980e634
13 changed files with 373 additions and 34 deletions

View File

@ -67,6 +67,11 @@ class TestPowerStoreDriver(test.TestCase):
configuration=self.configuration
)
self.fc_driver.do_setup({})
self._override_shared_conf("powerstore_nvme", override=True)
self.nvme_driver = driver.PowerStoreDriver(
configuration=self.configuration
)
self.nvme_driver.do_setup({})
def _override_shared_conf(self, *args, **kwargs):
return self.override_config(*args,

View File

@ -30,6 +30,14 @@ class TestBase(powerstore.TestPowerStoreDriver):
self.assertRaises(exception.InvalidInput,
self.driver.check_for_setup_error)
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_array_version")
def test_configuration_nvme_not_supported(self, mock_version):
mock_version.return_value = "2.0.0.0"
self.nvme_driver.do_setup({})
self.assertRaises(exception.InvalidInput,
self.nvme_driver.check_for_setup_error)
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_chap_config")
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
@ -56,9 +64,13 @@ class TestBase(powerstore.TestPowerStoreDriver):
self.driver._update_volume_stats)
self.assertIn("Failed to query PowerStore metrics", error.msg)
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_array_version")
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_chap_config")
def test_configuration_with_replication(self, mock_chap):
def test_configuration_with_replication(self,
mock_chap,
mock_version):
replication_device = [
{
"backend_id": "repl_1",
@ -67,6 +79,7 @@ class TestBase(powerstore.TestPowerStoreDriver):
"san_password": "test_2"
}
]
mock_version.return_value = "3.0.0.0"
self._override_shared_conf("replication_device",
override=replication_device)
self.driver.do_setup({})
@ -91,9 +104,13 @@ class TestBase(powerstore.TestPowerStoreDriver):
self.assertIn("PowerStore driver does not support more than one "
"replication device.", error.msg)
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_array_version")
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_chap_config")
def test_configuration_with_replication_failed_over(self, mock_chap):
def test_configuration_with_replication_failed_over(self,
mock_chap,
mock_version):
replication_device = [
{
"backend_id": "repl_1",
@ -102,6 +119,7 @@ class TestBase(powerstore.TestPowerStoreDriver):
"san_password": "test_2"
}
]
mock_version.return_value = "3.0.0.0"
self._override_shared_conf("replication_device",
override=replication_device)
self.driver.do_setup({})

View File

@ -0,0 +1,103 @@
# Copyright (c) 2021 Dell Inc. or its subsidiaries.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
import uuid
import ddt
import six
from cinder.tests.unit import test
from cinder.tests.unit.volume.drivers.dell_emc.powerstore import MockResponse
from cinder.volume.drivers.dell_emc.powerstore import client
CLIENT_OPTIONS = {
"rest_ip": "127.0.0.1",
"rest_username": "fake_user",
"rest_password": "fake_password",
"verify_certificate": False,
"certificate_path": None
}
ISCSI_IP_POOL_RESP = [
{
"address": "1.2.3.4",
"ip_port": {
"target_iqn":
"iqn.2022-07.com.dell:dellemc-powerstore-fake-iqn-1"
},
},
{
"address": "5.6.7.8",
"ip_port": {
"target_iqn":
"iqn.2022-07.com.dell:dellemc-powerstore-fake-iqn-1"
},
},
]
NVME_IP_POOL_RESP = [
{
"address": "11.22.33.44"
},
{
"address": "55.66.77.88"
}
]
@ddt.ddt
class TestClient(test.TestCase):
def setUp(self):
super(TestClient, self).setUp()
self.client = client.PowerStoreClient(**CLIENT_OPTIONS)
self.fake_volume = six.text_type(uuid.uuid4())
@ddt.data(("iSCSI", ISCSI_IP_POOL_RESP),
("NVMe", NVME_IP_POOL_RESP))
@ddt.unpack
@mock.patch("requests.request")
def test_get_ip_pool_address(self, protocol, ip_pool, mock_request):
mock_request.return_value = MockResponse(ip_pool, rc=200)
response = self.client.get_ip_pool_address(protocol)
mock_request.assert_called_once()
self.assertEqual(response, ip_pool)
@mock.patch("requests.request")
def test_get_volume_nguid(self, mock_request):
mock_request.return_value = MockResponse(
content={
"nguid": "nguid.76e02b0999y439958ttf546800ea7fe8"
},
rc=200
)
self.assertEqual(self.client.get_volume_nguid(self.fake_volume),
"76e02b0999y439958ttf546800ea7fe8")
@mock.patch("requests.request")
def test_get_array_version(self, mock_request):
mock_request.return_value = MockResponse(
content=[
{
"release_version": "3.0.0.0",
}
],
rc=200
)
self.assertEqual(self.client.get_array_version(),
"3.0.0.0")

View File

@ -23,9 +23,11 @@ from cinder.volume.drivers.dell_emc.powerstore import client
class TestReplication(powerstore.TestPowerStoreDriver):
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_array_version")
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.client."
"PowerStoreClient.get_chap_config")
def setUp(self, mock_chap):
def setUp(self, mock_chap, mock_version):
super(TestReplication, self).setUp()
self.replication_backend_id = "repl_1"
replication_device = [
@ -38,6 +40,7 @@ class TestReplication(powerstore.TestPowerStoreDriver):
]
self._override_shared_conf("replication_device",
override=replication_device)
mock_version.return_value = "3.0.0.0"
self.driver.do_setup({})
self.driver.check_for_setup_error()
self.volume = fake_volume.fake_volume_obj(

View File

@ -31,6 +31,12 @@ class TestVolumeAttachDetach(powerstore.TestPowerStoreDriver):
mock_chap.return_value = {"mode": "Single"}
self.iscsi_driver.check_for_setup_error()
self.fc_driver.check_for_setup_error()
with mock.patch.object(self.nvme_driver.adapter.client,
"get_array_version",
return_value=(
"3.0.0.0"
)):
self.nvme_driver.check_for_setup_error()
self.volume = fake_volume.fake_volume_obj(
self.context,
host="host@backend",
@ -74,6 +80,20 @@ class TestVolumeAttachDetach(powerstore.TestPowerStoreDriver):
"wwn": "58:cc:f0:98:49:23:07:02"
},
]
fake_nvme_portals_response = [
{
"address": "11.22.33.44"
},
{
"address": "55.66.77.88"
}
]
fake_nvme_nqn_response = [
{
"nvm_subsystem_nqn":
"nqn.2020-07.com.dell:powerstore:00:test-nqn"
}
]
self.fake_connector = {
"host": self.volume.host,
"wwpns": ["58:cc:f0:98:49:21:07:02", "58:cc:f0:98:49:23:07:02"],
@ -89,6 +109,16 @@ class TestVolumeAttachDetach(powerstore.TestPowerStoreDriver):
"get_fc_port",
return_value=fake_fc_wwns_response
)
self.nvme_portal_mock = self.mock_object(
self.nvme_driver.adapter.client,
"get_ip_pool_address",
return_value=fake_nvme_portals_response
)
self.nvme_nqn_mock = self.mock_object(
self.nvme_driver.adapter.client,
"get_subsystem_nqn",
return_value=fake_nvme_nqn_response
)
def test_initialize_connection_chap_enabled(self):
self.iscsi_driver.adapter.use_chap_auth = True
@ -160,6 +190,10 @@ class TestVolumeAttachDetach(powerstore.TestPowerStoreDriver):
self.assertIn("There are no accessible iSCSI targets on the system.",
error.msg)
def test_get_nvme_targets(self):
portals, nqn = self.nvme_driver.adapter._get_nvme_targets()
self.assertEqual(2, len(portals))
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.adapter."
"CommonAdapter._detach_volume_from_hosts")
@mock.patch("cinder.volume.drivers.dell_emc.powerstore.adapter."

View File

@ -18,7 +18,6 @@
from oslo_log import log as logging
from oslo_utils import strutils
from cinder.common import constants
from cinder import coordination
from cinder import exception
from cinder.i18n import _
@ -32,9 +31,8 @@ from cinder.volume import volume_utils
LOG = logging.getLogger(__name__)
PROTOCOL_FC = constants.FC
PROTOCOL_ISCSI = constants.ISCSI
CHAP_MODE_SINGLE = "Single"
POWERSTORE_NVME_VERSION_SUPPORT = "3.0"
class CommonAdapter(object):
@ -74,7 +72,7 @@ class CommonAdapter(object):
def check_for_setup_error(self):
self.client.check_for_setup_error()
if self.storage_protocol == PROTOCOL_ISCSI:
if self.storage_protocol == utils.PROTOCOL_ISCSI:
chap_config = self.client.get_chap_config()
if chap_config.get("mode") == CHAP_MODE_SINGLE:
self.use_chap_auth = True
@ -560,7 +558,7 @@ class CommonAdapter(object):
:param host: PowerStore host object
:param volume: OpenStack volume object
:return: attached volume logical number
:return: attached volume identifier
"""
provider_id = self._get_volume_provider_id(volume)
@ -575,21 +573,25 @@ class CommonAdapter(object):
"host_provider_id": host["id"],
})
self.client.attach_volume_to_host(host["id"], provider_id)
volume_lun = self.client.get_volume_lun(host["id"], provider_id)
if self.storage_protocol == utils.PROTOCOL_NVME:
volume_identifier = self.client.get_volume_nguid(provider_id)
else:
volume_identifier = self.client.get_volume_lun(host["id"],
provider_id)
LOG.debug("Successfully attached PowerStore volume %(volume_name)s "
"with id %(volume_id)s to host %(host_name)s. "
"PowerStore volume id: %(volume_provider_id)s, "
"host id: %(host_provider_id)s. Volume LUN: "
"%(volume_lun)s.",
"host id: %(host_provider_id)s. Volume identifier: "
"%(volume_identifier)s.",
{
"volume_name": volume.name,
"volume_id": volume.id,
"host_name": host["name"],
"volume_provider_id": provider_id,
"host_provider_id": host["id"],
"volume_lun": volume_lun,
"volume_identifier": volume_identifier,
})
return volume_lun
return volume_identifier
def _create_host_and_attach(self, connector, volume):
"""Create PowerStore host and attach volume.
@ -610,11 +612,13 @@ class CommonAdapter(object):
:return: volume connection properties
"""
chap_credentials, volume_lun = self._create_host_and_attach(
chap_credentials, volume_identifier = self._create_host_and_attach(
connector,
volume
)
connection_properties = self._get_connection_properties(volume_lun)
connection_properties = self._get_connection_properties(
volume_identifier
)
if self.use_chap_auth:
connection_properties["data"]["auth_method"] = "CHAP"
connection_properties["data"]["auth_username"] = (
@ -1019,7 +1023,7 @@ class CommonAdapter(object):
class FibreChannelAdapter(CommonAdapter):
def __init__(self, **kwargs):
super(FibreChannelAdapter, self).__init__(**kwargs)
self.storage_protocol = PROTOCOL_FC
self.storage_protocol = utils.PROTOCOL_FC
self.driver_volume_type = "fibre_channel"
@staticmethod
@ -1043,10 +1047,10 @@ class FibreChannelAdapter(CommonAdapter):
raise exception.VolumeBackendAPIException(data=msg)
return wwns
def _get_connection_properties(self, volume_lun):
def _get_connection_properties(self, volume_identifier):
"""Fill connection properties dict with data to attach volume.
:param volume_lun: attached volume logical unit number
:param volume_identifier: attached volume logical unit number
:return: connection properties
"""
@ -1055,7 +1059,7 @@ class FibreChannelAdapter(CommonAdapter):
"driver_volume_type": self.driver_volume_type,
"data": {
"target_discovered": False,
"target_lun": volume_lun,
"target_lun": volume_identifier,
"target_wwn": target_wwns,
}
}
@ -1064,7 +1068,7 @@ class FibreChannelAdapter(CommonAdapter):
class iSCSIAdapter(CommonAdapter):
def __init__(self, **kwargs):
super(iSCSIAdapter, self).__init__(**kwargs)
self.storage_protocol = PROTOCOL_ISCSI
self.storage_protocol = utils.PROTOCOL_ISCSI
self.driver_volume_type = "iscsi"
@staticmethod
@ -1079,7 +1083,9 @@ class iSCSIAdapter(CommonAdapter):
iqns = []
portals = []
ip_pool_addresses = self.client.get_ip_pool_address()
ip_pool_addresses = self.client.get_ip_pool_address(
self.storage_protocol
)
for address in ip_pool_addresses:
if self._port_is_allowed(address["address"]):
portals.append(
@ -1092,10 +1098,10 @@ class iSCSIAdapter(CommonAdapter):
raise exception.VolumeBackendAPIException(data=msg)
return iqns, portals
def _get_connection_properties(self, volume_lun):
def _get_connection_properties(self, volume_identifier):
"""Fill connection properties dict with data to attach volume.
:param volume_lun: attached volume logical unit number
:param volume_identifier: attached volume logical unit number
:return: connection properties
"""
@ -1106,9 +1112,74 @@ class iSCSIAdapter(CommonAdapter):
"target_discovered": False,
"target_portal": portals[0],
"target_iqn": iqns[0],
"target_lun": volume_lun,
"target_lun": volume_identifier,
"target_portals": portals,
"target_iqns": iqns,
"target_luns": [volume_lun] * len(portals),
"target_luns": [volume_identifier] * len(portals),
},
}
class NVMEoFAdapter(CommonAdapter):
def __init__(self, **kwargs):
super(NVMEoFAdapter, self).__init__(**kwargs)
self.storage_protocol = utils.PROTOCOL_NVME
self.driver_volume_type = "nvmeof"
@staticmethod
def initiators(connector):
return [connector["nqn"]]
def check_for_setup_error(self):
array_version = self.client.get_array_version()
if not utils.version_gte(
array_version,
POWERSTORE_NVME_VERSION_SUPPORT
):
msg = (_("PowerStore arrays support NVMe-OF starting from version "
"%(nvme_support_version)s. Current PowerStore array "
"version: %(current_version)s.")
% {"nvme_support_version": POWERSTORE_NVME_VERSION_SUPPORT,
"current_version": array_version, })
LOG.error(msg)
raise exception.InvalidInput(reason=msg)
super(NVMEoFAdapter, self).check_for_setup_error()
def _get_nvme_targets(self):
"""Get available NVMe portals and subsystem NQN.
:return: NVMe portals and NQN
"""
portals = []
ip_pool_addresses = self.client.get_ip_pool_address(
self.storage_protocol
)
for address in ip_pool_addresses:
if self._port_is_allowed(address["address"]):
portals.append(address["address"])
if not portals:
msg = _("There are no accessible NVMe targets on the "
"system.")
raise exception.VolumeBackendAPIException(data=msg)
nqn = self.client.get_subsystem_nqn()
return portals, nqn
def _get_connection_properties(self, volume_identifier):
"""Fill connection properties dict with data to attach volume.
:param volume_identifier: attached volume NGUID
:return: connection properties
"""
portals, nqn = self._get_nvme_targets()
return {
"driver_volume_type": self.driver_volume_type,
"data": {
"target_portal": portals[0],
"nqn": nqn,
"target_port": 4420,
"transport_type": "tcp",
"volume_nguid": volume_identifier
},
}

View File

@ -25,6 +25,7 @@ import requests
from cinder import exception
from cinder.i18n import _
from cinder import utils as cinder_utils
from cinder.volume.drivers.dell_emc.powerstore import utils
LOG = logging.getLogger(__name__)
@ -386,14 +387,40 @@ class PowerStoreClient(object):
raise exception.VolumeBackendAPIException(data=msg)
return response
def get_ip_pool_address(self):
def get_subsystem_nqn(self):
r, response = self._send_get_request(
"/ip_pool_address",
"/cluster",
params={
"select": "nvm_subsystem_nqn"
}
)
if r.status_code not in self.ok_codes:
msg = _("Failed to query PowerStore NVMe subsystem NQN.")
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
try:
nqn = response[0].get("nvm_subsystem_nqn")
return nqn
except IndexError:
msg = _("PowerStore NVMe subsystem NQN is not found.")
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
def get_ip_pool_address(self, protocol):
params = {}
if protocol == utils.PROTOCOL_ISCSI:
params = {
"purposes": "cs.{Storage_Iscsi_Target}",
"select": "address,ip_port(target_iqn)"
}
elif protocol == utils.PROTOCOL_NVME:
params = {
"purposes": "cs.{Storage_NVMe_TCP_Port}",
"select": "address"
}
r, response = self._send_get_request(
"/ip_pool_address",
params=params
)
if r.status_code not in self.ok_codes:
msg = _("Failed to query PowerStore IP pool addresses.")
@ -743,3 +770,32 @@ class PowerStoreClient(object):
% volume_id)
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
def get_array_version(self):
r, response = self._send_get_request(
"/software_installed",
params={
"select": "release_version",
"is_cluster": "eq.True"
}
)
if r.status_code not in self.ok_codes:
msg = _("Failed to query PowerStore array version.")
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
return response[0].get("release_version")
def get_volume_nguid(self, volume_id):
r, response = self._send_get_request(
"/volume/%s" % volume_id,
params={
"select": "nguid",
}
)
if r.status_code not in self.ok_codes:
msg = (_("Failed to query PowerStore volume with id %s.")
% volume_id)
LOG.error(msg)
raise exception.VolumeBackendAPIException(data=msg)
nguid = response["nguid"].split('.')[1]
return nguid

View File

@ -25,6 +25,7 @@ from cinder.volume import configuration
from cinder.volume import driver
from cinder.volume.drivers.dell_emc.powerstore import adapter
from cinder.volume.drivers.dell_emc.powerstore import options
from cinder.volume.drivers.dell_emc.powerstore import utils
from cinder.volume.drivers.san import san
from cinder.volume import manager
@ -50,9 +51,10 @@ class PowerStoreDriver(driver.VolumeDriver):
1.1.2 - Fix iSCSI targets not being returned from the REST API call if
targets are used for multiple purposes
(iSCSI target, Replication target, etc.)
1.2.0 - Add NVMe-OF support
"""
VERSION = "1.1.2"
VERSION = "1.2.0"
VENDOR = "Dell EMC"
# ThirdPartySystems wiki page
@ -89,9 +91,11 @@ class PowerStoreDriver(driver.VolumeDriver):
if not self.active_backend_id:
self.active_backend_id = manager.VolumeManager.FAILBACK_SENTINEL
storage_protocol = self.configuration.safe_get("storage_protocol")
if (
if self.configuration.safe_get(options.POWERSTORE_NVME):
adapter_class = adapter.NVMEoFAdapter
elif (
storage_protocol and
storage_protocol.lower() == adapter.PROTOCOL_FC.lower()
storage_protocol.lower() == utils.PROTOCOL_FC.lower()
):
adapter_class = adapter.FibreChannelAdapter
else:

View File

@ -19,6 +19,7 @@ from oslo_config import cfg
POWERSTORE_APPLIANCES = "powerstore_appliances"
POWERSTORE_PORTS = "powerstore_ports"
POWERSTORE_NVME = "powerstore_nvme"
POWERSTORE_OPTS = [
cfg.ListOpt(POWERSTORE_APPLIANCES,
@ -34,5 +35,9 @@ POWERSTORE_OPTS = [
default=[],
help="Allowed ports. Comma separated list of PowerStore "
"iSCSI IPs or FC WWNs (ex. 58:cc:f0:98:49:22:07:02) "
"to be used. If option is not set all ports are allowed.")
"to be used. If option is not set all ports are allowed."
),
cfg.BoolOpt(POWERSTORE_NVME,
default=False,
help="Connect PowerStore volumes using NVMe-OF.")
]

View File

@ -15,12 +15,14 @@
"""Utilities for Dell EMC PowerStore Cinder driver."""
from distutils import version
import functools
import re
from oslo_log import log as logging
from oslo_utils import units
from cinder.common import constants
from cinder import exception
from cinder.i18n import _
from cinder.objects import fields
@ -31,6 +33,9 @@ from cinder.volume import volume_utils
LOG = logging.getLogger(__name__)
CHAP_DEFAULT_USERNAME = "PowerStore_iSCSI_CHAP_Username"
CHAP_DEFAULT_SECRET_LENGTH = 60
PROTOCOL_FC = constants.FC
PROTOCOL_ISCSI = constants.ISCSI
PROTOCOL_NVME = "NVMe"
def bytes_to_gib(size_in_bytes):
@ -91,7 +96,7 @@ def powerstore_host_name(connector, protocol):
"""Generate PowerStore host name for connector.
:param connector: connection properties
:param protocol: storage protocol (FC or iSCSI)
:param protocol: storage protocol (FC, iSCSI or NVMe)
:return: unique host name
"""
@ -178,3 +183,7 @@ def is_group_a_cg_snapshot_type(func):
raise NotImplementedError
return func(self, *args, **kwargs)
return inner
def version_gte(ver1, ver2):
return version.LooseVersion(ver1) >= version.LooseVersion(ver2)

View File

@ -49,6 +49,33 @@ Add the following content into ``/etc/cinder/cinder.conf``:
# PowerStore allowed ports
powerstore_ports = <Allowed ports> # Ex. 58:cc:f0:98:49:22:07:02,58:cc:f0:98:49:23:07:02
Driver configuration to use NVMe-OF
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
NVMe-OF support was added in PowerStore starting from version 2.1.
.. note:: Currently the driver supports only NVMe over TCP.
To configure NVMe-OF driver add the following
content into ``/etc/cinder/cinder.conf``:
.. code-block:: ini
[DEFAULT]
enabled_backends = powerstore
[powerstore]
# PowerStore REST IP
san_ip = <San IP>
# PowerStore REST username and password
san_login = <San username>
san_password = <San Password>
# Volume driver name
volume_driver = cinder.volume.drivers.dell_emc.powerstore.driver.PowerStoreDriver
# Backend name
volume_backend_name = <Backend name>
powerstore_nvme = True
Driver options
~~~~~~~~~~~~~~

View File

@ -25,7 +25,7 @@ title=Dell XtremeIO Storage Driver (FC, iSCSI)
title=Dell PowerMax (2000, 8000) Storage Driver (iSCSI, FC)
[driver.dell_emc_powerstore]
title=Dell PowerStore Storage Driver (iSCSI, FC)
title=Dell PowerStore Storage Driver (iSCSI, FC, NVMe-TCP)
[driver.dell_emc_sc]
title=Dell SC Series Storage Driver (iSCSI, FC)

View File

@ -0,0 +1,4 @@
---
features:
- |
Dell PowerStore driver: Added NVMe-TCP support.