From 56eaf475a4f2e99c81682eaaf99454b245059de5 Mon Sep 17 00:00:00 2001 From: "Walter A. Boring IV" Date: Mon, 3 Jun 2019 19:07:09 +0000 Subject: [PATCH] Add ceph iscsi volume driver The driver requires the new rbd-iscsi-client package, which is used to talk to the rbd-target-api on the ceph iscsi gateway node. The rbd-target-api is a python script meant to keep ceph iscsi gw nodes in sync with each other, but the API is works for creating iscsi targets. This is a new driver that makes heavy use of the ceph-iscsi project's rbd-target-api python REST client here: https://github.com/ceph/ceph-iscsi The driver is a derivation of the rbd driver, and the intention is to reuse as much of the base rbd driver as possible and just do iSCSI specific code here. Change-Id: Iff0e4d1137851c8f0b8ec25632d1186c2859b2fc --- cinder/opts.py | 3 + .../unit/volume/drivers/ceph/__init__.py | 0 .../drivers/ceph/fake_rbd_iscsi_client.py | 25 + .../ceph/fake_rbd_iscsi_client_exceptions.py | 116 ++++ .../volume/drivers/ceph/test_rbd_iscsi.py | 246 +++++++++ cinder/volume/drivers/ceph/__init__.py | 0 cinder/volume/drivers/ceph/rbd_iscsi.py | 496 ++++++++++++++++++ cinder/volume/drivers/rbd.py | 3 +- doc/source/reference/support-matrix.ini | 14 + driver-requirements.txt | 3 + lower-constraints.txt | 1 + .../ceph-iscsi-driver-b515bd7fb73ce13b.yaml | 6 + setup.cfg | 3 + 13 files changed, 915 insertions(+), 1 deletion(-) create mode 100644 cinder/tests/unit/volume/drivers/ceph/__init__.py create mode 100644 cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client.py create mode 100644 cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client_exceptions.py create mode 100644 cinder/tests/unit/volume/drivers/ceph/test_rbd_iscsi.py create mode 100644 cinder/volume/drivers/ceph/__init__.py create mode 100644 cinder/volume/drivers/ceph/rbd_iscsi.py create mode 100644 releasenotes/notes/ceph-iscsi-driver-b515bd7fb73ce13b.yaml diff --git a/cinder/opts.py b/cinder/opts.py index 429ef21f6b3..6865281122a 100644 --- a/cinder/opts.py +++ b/cinder/opts.py @@ -69,6 +69,8 @@ from cinder import ssh_utils as cinder_sshutils from cinder.transfer import api as cinder_transfer_api from cinder.volume import api as cinder_volume_api from cinder.volume import driver as cinder_volume_driver +from cinder.volume.drivers.ceph import rbd_iscsi as \ + cinder_volume_drivers_ceph_rbdiscsi from cinder.volume.drivers.datera import datera_iscsi as \ cinder_volume_drivers_datera_dateraiscsi from cinder.volume.drivers.dell_emc.powerflex import driver as \ @@ -310,6 +312,7 @@ def list_opts(): cinder_volume_driver.scst_opts, cinder_volume_driver.image_opts, cinder_volume_driver.fqdn_opts, + cinder_volume_drivers_ceph_rbdiscsi.RBD_ISCSI_OPTS, cinder_volume_drivers_dell_emc_powerflex_driver. powerflex_opts, cinder_volume_drivers_dell_emc_powermax_common.powermax_opts, diff --git a/cinder/tests/unit/volume/drivers/ceph/__init__.py b/cinder/tests/unit/volume/drivers/ceph/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client.py b/cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client.py new file mode 100644 index 00000000000..3f655500465 --- /dev/null +++ b/cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Fake rbd-iscsi-client for testing without installing the client.""" + +import sys +from unittest import mock + +from cinder.tests.unit.volume.drivers.ceph \ + import fake_rbd_iscsi_client_exceptions as clientexceptions + +rbdclient = mock.MagicMock() +rbdclient.version = "0.1.5" +rbdclient.exceptions = clientexceptions + +sys.modules['rbd_iscsi_client'] = rbdclient diff --git a/cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client_exceptions.py b/cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client_exceptions.py new file mode 100644 index 00000000000..7a70ce755ec --- /dev/null +++ b/cinder/tests/unit/volume/drivers/ceph/fake_rbd_iscsi_client_exceptions.py @@ -0,0 +1,116 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +"""Fake client exceptions to use.""" + + +class UnsupportedVersion(Exception): + """Unsupported version of the client.""" + pass + + +class ClientException(Exception): + """The base exception class for these fake exceptions.""" + _error_code = None + _error_desc = None + _error_ref = None + + _debug1 = None + _debug2 = None + + def __init__(self, error=None): + if error: + if 'code' in error: + self._error_code = error['code'] + if 'desc' in error: + self._error_desc = error['desc'] + if 'ref' in error: + self._error_ref = error['ref'] + + if 'debug1' in error: + self._debug1 = error['debug1'] + if 'debug2' in error: + self._debug2 = error['debug2'] + + def get_code(self): + return self._error_code + + def get_description(self): + return self._error_desc + + def get_ref(self): + return self._error_ref + + def __str__(self): + formatted_string = self.message + if self.http_status: + formatted_string += " (HTTP %s)" % self.http_status + if self._error_code: + formatted_string += " %s" % self._error_code + if self._error_desc: + formatted_string += " - %s" % self._error_desc + if self._error_ref: + formatted_string += " - %s" % self._error_ref + + if self._debug1: + formatted_string += " (1: '%s')" % self._debug1 + + if self._debug2: + formatted_string += " (2: '%s')" % self._debug2 + + return formatted_string + + +class HTTPConflict(ClientException): + http_status = 409 + message = "Conflict" + + def __init__(self, error=None): + if error: + super(HTTPConflict, self).__init__(error) + if 'message' in error: + self._error_desc = error['message'] + + def get_description(self): + return self._error_desc + + +class HTTPNotFound(ClientException): + http_status = 404 + message = "Not found" + + +class HTTPForbidden(ClientException): + http_status = 403 + message = "Forbidden" + + +class HTTPBadRequest(ClientException): + http_status = 400 + message = "Bad request" + + +class HTTPUnauthorized(ClientException): + http_status = 401 + message = "Unauthorized" + + +class HTTPServerError(ClientException): + http_status = 500 + message = "Error" + + def __init__(self, error=None): + if error and 'message' in error: + self._error_desc = error['message'] + + def get_description(self): + return self._error_desc diff --git a/cinder/tests/unit/volume/drivers/ceph/test_rbd_iscsi.py b/cinder/tests/unit/volume/drivers/ceph/test_rbd_iscsi.py new file mode 100644 index 00000000000..d593345b69b --- /dev/null +++ b/cinder/tests/unit/volume/drivers/ceph/test_rbd_iscsi.py @@ -0,0 +1,246 @@ +# Copyright 2012 Josh Durgin +# Copyright 2013 Canonical Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from unittest import mock + +import ddt + +from cinder import context +from cinder import exception +from cinder.tests.unit import fake_constants as fake +from cinder.tests.unit import fake_volume +from cinder.tests.unit import test +from cinder.tests.unit.volume.drivers.ceph \ + import fake_rbd_iscsi_client as fake_client +import cinder.volume.drivers.ceph.rbd_iscsi as driver + +# This is used to collect raised exceptions so that tests may check what was +# raised. +# NOTE: this must be initialised in test setUp(). +RAISED_EXCEPTIONS = [] + + +@ddt.ddt +class RBDISCSITestCase(test.TestCase): + + def setUp(self): + global RAISED_EXCEPTIONS + RAISED_EXCEPTIONS = [] + super(RBDISCSITestCase, self).setUp() + + self.context = context.get_admin_context() + + # bogus access to prevent pep8 violation + # from the import of fake_client. + # fake_client must be imported to create the fake + # rbd_iscsi_client system module + fake_client.rbdclient + + self.fake_target_iqn = 'iqn.2019-01.com.suse.iscsi-gw:iscsi-igw' + self.fake_valid_response = {'status': '200'} + + self.fake_clients = \ + {'response': + {'Content-Type': 'application/json', + 'Content-Length': '55', + 'Server': 'Werkzeug/0.14.1 Python/2.7.15rc1', + 'Date': 'Wed, 19 Jun 2019 20:13:18 GMT', + 'status': '200', + 'content-location': 'http://192.168.121.11:5001/api/clients/' + 'XX_REPLACE_ME'}, + 'body': + {'clients': ['iqn.1993-08.org.debian:01:5d3b9abba13d']}} + + self.volume_a = fake_volume.fake_volume_obj( + self.context, + **{'name': u'volume-0000000a', + 'id': '4c39c3c7-168f-4b32-b585-77f1b3bf0a38', + 'size': 10}) + + self.volume_b = fake_volume.fake_volume_obj( + self.context, + **{'name': u'volume-0000000b', + 'id': '0c7d1f44-5a06-403f-bb82-ae7ad0d693a6', + 'size': 10}) + + self.volume_c = fake_volume.fake_volume_obj( + self.context, + **{'name': u'volume-0000000a', + 'id': '55555555-222f-4b32-b585-9991b3bf0a99', + 'size': 12, + 'encryption_key_id': fake.ENCRYPTION_KEY_ID}) + + def setup_configuration(self): + config = mock.MagicMock() + config.rbd_cluster_name = 'nondefault' + config.rbd_pool = 'rbd' + config.rbd_ceph_conf = '/etc/ceph/my_ceph.conf' + config.rbd_secret_uuid = None + config.rbd_user = 'cinder' + config.volume_backend_name = None + config.rbd_iscsi_api_user = 'fake_user' + config.rbd_iscsi_api_password = 'fake_password' + config.rbd_iscsi_api_url = 'http://fake.com:5000' + return config + + @mock.patch( + 'rbd_iscsi_client.client.RBDISCSIClient', + spec=True, + ) + def setup_mock_client(self, _m_client, config=None, mock_conf=None): + _m_client = _m_client.return_value + + # Configure the base constants, defaults etc... + if mock_conf: + _m_client.configure_mock(**mock_conf) + + if config is None: + config = self.setup_configuration() + + self.driver = driver.RBDISCSIDriver(configuration=config) + self.driver.set_initialized() + return _m_client + + @mock.patch('rbd_iscsi_client.version', '0.1.0') + def test_unsupported_client_version(self): + self.setup_mock_client() + with mock.patch('cinder.volume.drivers.rbd.RBDDriver.do_setup'): + self.assertRaises(exception.InvalidInput, + self.driver.do_setup, None) + + @ddt.data({'user': None, 'password': 'foo', + 'url': 'http://fake.com:5000', 'iqn': None}, + {'user': None, 'password': None, + 'url': 'http://fake', 'iqn': None}, + {'user': None, 'password': None, + 'url': None, 'iqn': None}, + {'user': 'fake', 'password': 'fake', + 'url': None, 'iqn': None}, + {'user': 'fake', 'password': 'fake', + 'url': 'fake', 'iqn': None}, + ) + @ddt.unpack + def test_min_config(self, user, password, url, iqn): + config = self.setup_configuration() + config.rbd_iscsi_api_user = user + config.rbd_iscsi_api_password = password + config.rbd_iscsi_api_url = url + config.rbd_iscsi_target_iqn = iqn + self.setup_mock_client(config=config) + + with mock.patch('cinder.volume.drivers.rbd.RBDDriver' + '.check_for_setup_error'): + self.assertRaises(exception.InvalidConfigurationValue, + self.driver.check_for_setup_error) + + @ddt.data({'response': None}, + {'response': {'nothing': 'nothing'}}, + {'response': {'status': '300'}}) + @ddt.unpack + def test_do_setup(self, response): + mock_conf = { + 'get_api.return_value': (response, None)} + mock_client = self.setup_mock_client(mock_conf=mock_conf) + + with mock.patch('cinder.volume.drivers.rbd.RBDDriver.do_setup'), \ + mock.patch.object(driver.RBDISCSIDriver, + '_create_client') as mock_create_client: + mock_create_client.return_value = mock_client + self.assertRaises(exception.InvalidConfigurationValue, + self.driver.do_setup, None) + + @mock.patch('rbd_iscsi_client.version', "0.1.4") + def test_unsupported_version(self): + self.setup_mock_client() + self.assertRaises(exception.InvalidInput, + self.driver._create_client) + + @ddt.data({'status': '200', + 'target_iqn': 'iqn.2019-01.com.suse.iscsi-gw:iscsi-igw', + 'clients': ['foo']}, + {'status': '300', + 'target_iqn': 'iqn.2019-01.com.suse.iscsi-gw:iscsi-igw', + 'clients': None} + ) + @ddt.unpack + def test__get_clients(self, status, target_iqn, clients): + config = self.setup_configuration() + config.rbd_iscsi_target_iqn = target_iqn + + response = self.fake_clients['response'] + response['status'] = status + response['content-location'] = ( + response['content-location'].replace('XX_REPLACE_ME', target_iqn)) + + body = self.fake_clients['body'] + mock_conf = { + 'get_clients.return_value': (response, body), + 'get_api.return_value': (self.fake_valid_response, None) + } + mock_client = self.setup_mock_client(mock_conf=mock_conf, + config=config) + + with mock.patch('cinder.volume.drivers.rbd.RBDDriver.do_setup'), \ + mock.patch.object(driver.RBDISCSIDriver, + '_create_client') as mock_create_client: + mock_create_client.return_value = mock_client + self.driver.do_setup(None) + if status == '200': + actual_response = self.driver._get_clients() + self.assertEqual(actual_response, body) + else: + # we expect an exception + self.assertRaises(exception.VolumeBackendAPIException, + self.driver._get_clients) + + @ddt.data({'status': '200', + 'body': {'created': 'someday', + 'discovery_auth': 'somecrap', + 'disks': 'fakedisks', + 'gateways': 'fakegws', + 'targets': 'faketargets'}}, + {'status': '300', + 'body': None}) + @ddt.unpack + def test__get_config(self, status, body): + config = self.setup_configuration() + config.rbd_iscsi_target_iqn = self.fake_target_iqn + + response = self.fake_clients['response'] + response['status'] = status + response['content-location'] = ( + response['content-location'].replace('XX_REPLACE_ME', + self.fake_target_iqn)) + + mock_conf = { + 'get_config.return_value': (response, body), + 'get_api.return_value': (self.fake_valid_response, None) + } + mock_client = self.setup_mock_client(mock_conf=mock_conf, + config=config) + + with mock.patch('cinder.volume.drivers.rbd.RBDDriver.do_setup'), \ + mock.patch.object(driver.RBDISCSIDriver, + '_create_client') as mock_create_client: + mock_create_client.return_value = mock_client + self.driver.do_setup(None) + if status == '200': + actual_response = self.driver._get_config() + self.assertEqual(body, actual_response) + else: + # we expect an exception + self.assertRaises(exception.VolumeBackendAPIException, + self.driver._get_config) diff --git a/cinder/volume/drivers/ceph/__init__.py b/cinder/volume/drivers/ceph/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/cinder/volume/drivers/ceph/rbd_iscsi.py b/cinder/volume/drivers/ceph/rbd_iscsi.py new file mode 100644 index 00000000000..3c99ce13e5e --- /dev/null +++ b/cinder/volume/drivers/ceph/rbd_iscsi.py @@ -0,0 +1,496 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""RADOS Block Device iSCSI Driver""" + +from distutils import version + +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import netutils + +from cinder import exception +from cinder.i18n import _ +from cinder import interface +from cinder import utils +from cinder.volume import configuration +from cinder.volume.drivers import rbd +from cinder.volume import volume_utils + +try: + import rbd_iscsi_client + from rbd_iscsi_client import client + from rbd_iscsi_client import exceptions as client_exceptions +except ImportError: + rbd_iscsi_client = None + client = None + client_exceptions = None + +LOG = logging.getLogger(__name__) + +RBD_ISCSI_OPTS = [ + cfg.StrOpt('rbd_iscsi_api_user', + default='', + help='The username for the rbd_target_api service'), + cfg.StrOpt('rbd_iscsi_api_password', + default='', + secret=True, + help='The username for the rbd_target_api service'), + cfg.StrOpt('rbd_iscsi_api_url', + default='', + help='The url to the rbd_target_api service'), + cfg.BoolOpt('rbd_iscsi_api_debug', + default=False, + help='Enable client request debugging.'), + cfg.StrOpt('rbd_iscsi_target_iqn', + default=None, + help='The preconfigured target_iqn on the iscsi gateway.'), +] + +CONF = cfg.CONF +CONF.register_opts(RBD_ISCSI_OPTS, group=configuration.SHARED_CONF_GROUP) + + +MIN_CLIENT_VERSION = "0.1.8" + + +@interface.volumedriver +class RBDISCSIDriver(rbd.RBDDriver): + """Implements RADOS block device (RBD) iSCSI volume commands.""" + + VERSION = '1.0.0' + + # ThirdPartySystems wiki page + CI_WIKI_NAME = "Cinder_Jenkins" + + SUPPORTS_ACTIVE_ACTIVE = True + + STORAGE_PROTOCOL = 'iSCSI' + CHAP_LENGTH = 16 + + # The target IQN to use for creating all exports + # we map all the targets for OpenStack attaches to this. + target_iqn = None + + def __init__(self, active_backend_id=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.configuration.append_config_values(RBD_ISCSI_OPTS) + + @classmethod + def get_driver_options(cls): + additional_opts = cls._get_oslo_driver_opts( + 'replication_device', 'reserved_percentage', + 'max_over_subscription_ratio', 'volume_dd_blocksize', + 'driver_ssl_cert_verify', 'suppress_requests_ssl_warnings') + return rbd.RBD_OPTS + RBD_ISCSI_OPTS + additional_opts + + def _create_client(self): + client_version = rbd_iscsi_client.version + if (version.StrictVersion(client_version) < + version.StrictVersion(MIN_CLIENT_VERSION)): + ex_msg = (_('Invalid rbd_iscsi_client version found (%(found)s). ' + 'Version %(min)s or greater required. Run "pip' + ' install --upgrade rbd-iscsi-client" to upgrade' + ' the client.') + % {'found': client_version, + 'min': MIN_CLIENT_VERSION}) + LOG.error(ex_msg) + raise exception.InvalidInput(reason=ex_msg) + + config = self.configuration + ssl_warn = config.safe_get('suppress_requests_ssl_warnings') + cl = client.RBDISCSIClient( + config.safe_get('rbd_iscsi_api_user'), + config.safe_get('rbd_iscsi_api_password'), + config.safe_get('rbd_iscsi_api_url'), + secure=config.safe_get('driver_ssl_cert_verify'), + suppress_ssl_warnings=ssl_warn + ) + + return cl + + def _is_status_200(self, response): + return (response and 'status' in response and + response['status'] == '200') + + def do_setup(self, context): + """Perform initialization steps that could raise exceptions.""" + super(RBDISCSIDriver, self).do_setup(context) + if client is None: + msg = _("You must install rbd-iscsi-client python package " + "before using this driver.") + raise exception.VolumeDriverException(data=msg) + + # Make sure we have the basic settings we need to talk to the + # iscsi api service + config = self.configuration + self.client = self._create_client() + self.client.set_debug_flag(config.safe_get('rbd_iscsi_api_debug')) + resp, body = self.client.get_api() + if not self._is_status_200(resp): + # failed to fetch the open api url + raise exception.InvalidConfigurationValue( + option='rbd_iscsi_api_url', + value='Could not talk to the rbd-target-api') + + # The admin had to have setup a target_iqn in the iscsi gateway + # already in order for the gateways to work properly + self.target_iqn = self.configuration.safe_get('rbd_iscsi_target_iqn') + LOG.info("Using target_iqn '%s'", self.target_iqn) + + def check_for_setup_error(self): + """Return an error if prerequisites aren't met.""" + super(RBDISCSIDriver, self).check_for_setup_error() + + required_options = ['rbd_iscsi_api_user', + 'rbd_iscsi_api_password', + 'rbd_iscsi_api_url', + 'rbd_iscsi_target_iqn'] + + for attr in required_options: + val = getattr(self.configuration, attr) + if not val: + raise exception.InvalidConfigurationValue(option=attr, + value=val) + + def _get_clients(self): + # make sure we have + resp, body = self.client.get_clients(self.target_iqn) + if not self._is_status_200(resp): + msg = _("Failed to get_clients() from rbd-target-api") + raise exception.VolumeBackendAPIException(data=msg) + return body + + def _get_config(self): + resp, body = self.client.get_config() + if not self._is_status_200(resp): + msg = _("Failed to get_config() from rbd-target-api") + raise exception.VolumeBackendAPIException(data=msg) + return body + + def _get_disks(self): + resp, disks = self.client.get_disks() + if not self._is_status_200(resp): + msg = _("Failed to get_disks() from rbd-target-api") + raise exception.VolumeBackendAPIException(data=msg) + return disks + + def create_client(self, initiator_iqn): + """Create a client iqn on the gateway if it doesn't exist.""" + client = self._get_target_client(initiator_iqn) + if not client: + try: + self.client.create_client(self.target_iqn, + initiator_iqn) + except client_exceptions.ClientException as ex: + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + def _get_target_client(self, initiator_iqn): + """Get the config information for a client defined to a target.""" + config = self._get_config() + target_config = config['targets'][self.target_iqn] + if initiator_iqn in target_config['clients']: + return target_config['clients'][initiator_iqn] + + def _get_auth_for_client(self, initiator_iqn): + initiator_config = self._get_target_client(initiator_iqn) + if initiator_config: + auth = initiator_config['auth'] + return auth + + def _set_chap_for_client(self, initiator_iqn, username, password): + """Save the CHAP creds in the client on the gateway.""" + # username is 8-64 chars + # Password has to be 12-16 chars + LOG.debug("Setting chap creds to %(user)s : %(pass)s", + {'user': username, 'pass': password}) + try: + self.client.set_client_auth(self.target_iqn, + initiator_iqn, + username, + password) + except client_exceptions.ClientException as ex: + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + def _get_lun(self, iscsi_config, lun_name, initiator_iqn): + lun = None + target_info = iscsi_config['targets'][self.target_iqn] + luns = target_info['clients'][initiator_iqn]['luns'] + if lun_name in luns: + lun = {'name': lun_name, + 'id': luns[lun_name]['lun_id']} + + return lun + + def _lun_name(self, volume_name): + """Build the iscsi gateway lun name.""" + return ("%(pool)s/%(volume_name)s" % + {'pool': self.configuration.rbd_pool, + 'volume_name': volume_name}) + + def get_existing_disks(self): + """Get the existing list of registered volumes on the gateway.""" + resp, disks = self.client.get_disks() + return disks['disks'] + + @utils.trace + def create_disk(self, volume_name): + """Register the volume with the iscsi gateways. + + We have to register the volume with the iscsi gateway. + Exporting the volume won't work unless the gateway knows + about it. + """ + try: + self.client.find_disk(self.configuration.rbd_pool, + volume_name) + except client_exceptions.HTTPNotFound: + try: + # disk isn't known by the gateways, so lets add it. + self.client.create_disk(self.configuration.rbd_pool, + volume_name) + except client_exceptions.ClientException as ex: + LOG.exception("Couldn't create the disk entry to " + "export the volume.") + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + @utils.trace + def register_disk(self, target_iqn, volume_name): + """Register the disk with the target_iqn.""" + lun_name = self._lun_name(volume_name) + try: + self.client.register_disk(target_iqn, lun_name) + except client_exceptions.HTTPBadRequest as ex: + desc = ex.get_description() + search_str = ('is already mapped on target %(target_iqn)s' % + {'target_iqn': self.target_iqn}) + if desc.find(search_str): + # The volume is already registered + return + else: + LOG.error("Couldn't register the volume to the target_iqn") + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + except client_exceptions.ClientException as ex: + LOG.exception("Couldn't register the volume to the target_iqn", + ex) + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + @utils.trace + def unregister_disk(self, target_iqn, volume_name): + """Unregister the volume from the gateway.""" + lun_name = self._lun_name(volume_name) + try: + self.client.unregister_disk(target_iqn, lun_name) + except client_exceptions.ClientException as ex: + LOG.exception("Couldn't unregister the volume to the target_iqn", + ex) + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + @utils.trace + def export_disk(self, initiator_iqn, volume_name, iscsi_config): + """Export a volume to an initiator.""" + lun_name = self._lun_name(volume_name) + LOG.debug("Export lun %(lun)s", {'lun': lun_name}) + lun = self._get_lun(iscsi_config, lun_name, initiator_iqn) + if lun: + LOG.debug("Found existing lun export.") + return lun + + try: + LOG.debug("Creating new lun export for %(lun)s", + {'lun': lun_name}) + self.client.export_disk(self.target_iqn, initiator_iqn, + self.configuration.rbd_pool, + volume_name) + + resp, iscsi_config = self.client.get_config() + return self._get_lun(iscsi_config, lun_name, initiator_iqn) + except client_exceptions.ClientException as ex: + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + @utils.trace + def unexport_disk(self, initiator_iqn, volume_name, iscsi_config): + """Remove a volume from an initiator.""" + lun_name = self._lun_name(volume_name) + LOG.debug("unexport lun %(lun)s", {'lun': lun_name}) + lun = self._get_lun(iscsi_config, lun_name, initiator_iqn) + if not lun: + LOG.debug("Didn't find LUN on gateway.") + return + + try: + LOG.debug("unexporting %(lun)s", {'lun': lun_name}) + self.client.unexport_disk(self.target_iqn, initiator_iqn, + self.configuration.rbd_pool, + volume_name) + except client_exceptions.ClientException as ex: + LOG.exception(ex) + raise exception.VolumeBackendAPIException( + data=ex.get_description()) + + def find_client_luns(self, target_iqn, client_iqn, iscsi_config): + """Find luns already exported to an initiator.""" + if 'targets' in iscsi_config: + if target_iqn in iscsi_config['targets']: + target_info = iscsi_config['targets'][target_iqn] + if 'clients' in target_info: + clients = target_info['clients'] + client = clients[client_iqn] + luns = client['luns'] + return luns + + @utils.trace + def initialize_connection(self, volume, connector): + """Export a volume to a host.""" + # create client + initiator_iqn = connector['initiator'] + self.create_client(initiator_iqn) + auth = self._get_auth_for_client(initiator_iqn) + username = initiator_iqn + if not auth['password']: + password = volume_utils.generate_password(length=self.CHAP_LENGTH) + self._set_chap_for_client(initiator_iqn, username, password) + else: + LOG.debug("using existing CHAP password") + password = auth['password'] + + # add disk for export + iscsi_config = self._get_config() + + # First have to ensure that the disk is registered with + # the gateways. + self.create_disk(volume.name) + self.register_disk(self.target_iqn, volume.name) + + iscsi_config = self._get_config() + # Now export the disk to the initiator + lun = self.export_disk(initiator_iqn, volume.name, iscsi_config) + + # fetch the updated config so we can get the lun id + iscsi_config = self._get_config() + target_info = iscsi_config['targets'][self.target_iqn] + ips = target_info['ip_list'] + + target_portal = ips[0] + if netutils.is_valid_ipv6(target_portal): + target_portal = "[{}]:{}".format( + target_portal, "3260") + else: + target_portal = "{}:3260".format(target_portal) + + data = { + 'driver_volume_type': 'iscsi', + 'data': { + 'target_iqn': self.target_iqn, + 'target_portal': target_portal, + 'target_lun': lun['id'], + 'auth_method': 'CHAP', + 'auth_username': username, + 'auth_password': password, + } + } + return data + + def _delete_disk(self, volume): + """Remove the defined disk from the gateway.""" + + # We only do this when we know it's not exported + # anywhere in the gateway + lun_name = self._lun_name(volume.name) + config = self._get_config() + + # Now look for the disk on any exported target + found = False + for target_iqn in config['targets']: + # Do we have the volume we are looking for? + target = config['targets'][target_iqn] + for client_iqn in target['clients'].keys(): + if lun_name in target['clients'][client_iqn]['luns']: + found = True + + if not found: + # we can delete the disk definition + LOG.info("Deleteing volume definition in iscsi gateway for {}". + format(lun_name)) + self.client.delete_disk(self.configuration.rbd_pool, volume.name, + preserve_image=True) + + def _terminate_connection(self, volume, initiator_iqn, target_iqn, + iscsi_config): + # remove the disk from the client. + self.unexport_disk(initiator_iqn, volume.name, iscsi_config) + + # Try to unregister the disk, since nobody is using it. + self.unregister_disk(self.target_iqn, volume.name) + + config = self._get_config() + + # If there are no more luns exported to this initiator + # then delete the initiator + luns = self.find_client_luns(target_iqn, initiator_iqn, config) + if not luns: + LOG.debug("There aren't any more LUNs attached to %(iqn)s." + "So we unregister the volume and delete " + "the client entry", + {'iqn': initiator_iqn}) + + try: + self.client.delete_client(target_iqn, initiator_iqn) + except client_exceptions.ClientException: + LOG.warning("Tried to delete initiator %(iqn)s, but delete " + "failed.", {'iqns': initiator_iqn}) + + def _terminate_all(self, volume, iscsi_config): + """Find all exports of this volume for our target_iqn and detach.""" + disks = self._get_disks() + lun_name = self._lun_name(volume.name) + if lun_name not in disks['disks']: + LOG.debug("Volume {} not attached anywhere.".format( + lun_name + )) + return + + for target_iqn_tmp in iscsi_config['targets']: + if self.target_iqn != target_iqn_tmp: + # We don't touch exports for targets + # we aren't configured to manage. + continue + + target = iscsi_config['targets'][self.target_iqn] + for client_iqn in target['clients'].keys(): + if lun_name in target['clients'][client_iqn]['luns']: + self._terminate_connection(volume, client_iqn, + self.target_iqn, + iscsi_config) + + self._delete_disk(volume) + + @utils.trace + def terminate_connection(self, volume, connector, **kwargs): + """Unexport the volume from the gateway.""" + iscsi_config = self._get_config() + + if not connector: + # No connector was passed in, so this is a force detach + # we need to detach the volume from the configured target_iqn. + self._terminate_all(volume, iscsi_config) + + initiator_iqn = connector['initiator'] + self._terminate_connection(volume, initiator_iqn, self.target_iqn, + iscsi_config) + self._delete_disk(volume) diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py index 068821c1caf..9ee14836491 100644 --- a/cinder/volume/drivers/rbd.py +++ b/cinder/volume/drivers/rbd.py @@ -224,6 +224,7 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, RBD_FEATURE_OBJECT_MAP = 8 RBD_FEATURE_FAST_DIFF = 16 RBD_FEATURE_JOURNALING = 64 + STORAGE_PROTOCOL = 'ceph' def __init__(self, active_backend_id=None, *args, **kwargs): super(RBDDriver, self).__init__(*args, **kwargs) @@ -608,7 +609,7 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, stats = { 'vendor_name': 'Open Source', 'driver_version': self.VERSION, - 'storage_protocol': 'ceph', + 'storage_protocol': self.STORAGE_PROTOCOL, 'total_capacity_gb': 'unknown', 'free_capacity_gb': 'unknown', 'reserved_percentage': ( diff --git a/doc/source/reference/support-matrix.ini b/doc/source/reference/support-matrix.ini index 123f4d067b0..6150f9da038 100644 --- a/doc/source/reference/support-matrix.ini +++ b/doc/source/reference/support-matrix.ini @@ -162,6 +162,9 @@ title=Quobyte Storage Driver (quobyte) [driver.rbd] title=RBD (Ceph) Storage Driver (RBD) +[driver.rbd_iscsi] +title=(Ceph) iSCSI Storage Driver (iSCSI) + [driver.sandstone] title=SandStone Storage Driver (iSCSI) @@ -257,6 +260,7 @@ driver.pure=complete driver.qnap=complete driver.quobyte=complete driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=complete driver.storpool=complete @@ -324,6 +328,7 @@ driver.pure=complete driver.qnap=complete driver.quobyte=missing driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=complete driver.storpool=complete @@ -391,6 +396,7 @@ driver.pure=missing driver.qnap=missing driver.quobyte=missing driver.rbd=missing +driver.rbd_iscsi=missing driver.sandstone=missing driver.seagate=missing driver.storpool=missing @@ -461,6 +467,7 @@ driver.pure=missing driver.qnap=missing driver.quobyte=missing driver.rbd=missing +driver.rbd_iscsi=missing driver.sandstone=complete driver.seagate=missing driver.storpool=missing @@ -530,6 +537,7 @@ driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=missing driver.storpool=complete @@ -600,6 +608,7 @@ driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=missing +driver.rbd_iscsi=missing driver.sandstone=missing driver.seagate=missing driver.storpool=missing @@ -669,6 +678,7 @@ driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=missing driver.storpool=complete @@ -739,6 +749,7 @@ driver.pure=missing driver.qnap=missing driver.quobyte=missing driver.rbd=missing +driver.rbd_iscsi=missing driver.sandstone=missing driver.seagate=missing driver.storpool=complete @@ -809,6 +820,7 @@ driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=complete driver.storpool=complete @@ -876,6 +888,7 @@ driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=missing driver.storpool=missing @@ -947,6 +960,7 @@ driver.pure=complete driver.qnap=missing driver.quobyte=missing driver.rbd=complete +driver.rbd_iscsi=complete driver.sandstone=complete driver.seagate=missing driver.storpool=missing diff --git a/driver-requirements.txt b/driver-requirements.txt index 2330f305bb9..307153250c4 100644 --- a/driver-requirements.txt +++ b/driver-requirements.txt @@ -27,6 +27,9 @@ pyxcli>=1.1.5 # Apache-2.0 rados # LGPLv2.1 rbd # LGPLv2.1 +# RBD-iSCSI +rbd-iscsi-client # Apache-2.0 + # Dell EMC VNX and Unity storops>=1.2.3 # Apache-2.0 diff --git a/lower-constraints.txt b/lower-constraints.txt index 9ffc24bde66..435f583b6e1 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -104,6 +104,7 @@ python-swiftclient==3.10.1 pytz==2020.1 pyudev==0.22.0 PyYAML==5.3.1 +rbd-iscsi-client==0.1.8 reno==3.2.0 repoze.lru==0.7 requests==2.23.0 diff --git a/releasenotes/notes/ceph-iscsi-driver-b515bd7fb73ce13b.yaml b/releasenotes/notes/ceph-iscsi-driver-b515bd7fb73ce13b.yaml new file mode 100644 index 00000000000..2cc4093b458 --- /dev/null +++ b/releasenotes/notes/ceph-iscsi-driver-b515bd7fb73ce13b.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Added new Ceph iSCSI driver rbd_iscsi. This new driver is derived from + the rbd driver and allows all the same features as the rbd driver. + The only difference is that volume attachments are done via iSCSI. diff --git a/setup.cfg b/setup.cfg index 30735935dee..81bc515ef35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -92,6 +92,7 @@ all = storpool>=4.0.0 # Apache-2.0 storpool.spopenstack>=2.2.1 # Apache-2.0 dfs-sdk>=1.2.25 # Apache-2.0 + rbd-iscsi-client>=0.1.8 # Apache-2.0 datacore = websocket-client>=0.32.0 # LGPLv2+ powermax = @@ -119,6 +120,8 @@ storpool = storpool.spopenstack>=2.2.1 # Apache-2.0 datera = dfs-sdk>=1.2.25 # Apache-2.0 +rbd_iscsi = + rbd-iscsi-client>=0.1.8 # Apache-2.0 [mypy]