From f1bb51c25138a1aaab45b64e2934c0468b941677 Mon Sep 17 00:00:00 2001 From: Danny Webb Date: Wed, 1 Dec 2021 13:48:10 +0000 Subject: [PATCH] RBD backend QoS implementation QoS support for the Ceph Cinder driver - Support injecting QoS metadata into ceph when creating a volume - Supports updating QoS parameters when retype operation is performed Note(s): 1) The version history added to cinder/volume/drivers/rbd.py is incomplete due to lack of prior knowledge in regards to the driver versioning. Signed-off-by: Danny Webb Co-Authored-By: Sergey Drozdov Implements: rbd-backend-qos Blueprint: https://blueprints.launchpad.net/cinder/+spec/rbd-backend-qos Change-Id: I25862085074d15e6cebb7f69c258fa9bcafe6d59 --- cinder/tests/unit/volume/drivers/test_rbd.py | 248 +++++++++++++++++- cinder/volume/drivers/rbd.py | 197 +++++++++++++- .../drivers/ceph-rbd-volume-driver.rst | 64 +++++ ...d-qos-implementation-0e141b742e277d26.yaml | 4 + 4 files changed, 499 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/rbd-backend-qos-implementation-0e141b742e277d26.yaml diff --git a/cinder/tests/unit/volume/drivers/test_rbd.py b/cinder/tests/unit/volume/drivers/test_rbd.py index c722cfc572a..06dca12054b 100644 --- a/cinder/tests/unit/volume/drivers/test_rbd.py +++ b/cinder/tests/unit/volume/drivers/test_rbd.py @@ -44,9 +44,9 @@ from cinder.tests.unit import utils from cinder.tests.unit.volume import test_driver from cinder.volume import configuration as conf import cinder.volume.drivers.rbd as driver +from cinder.volume import qos_specs from cinder.volume import volume_utils - # This is used to collect raised exceptions so that tests may check what was # raised. # NOTE: this must be initialised in test setUp(). @@ -266,6 +266,11 @@ class RBDTestCase(test.TestCase): 'host': 'host@fakebackend#fakepool'} }) + self.qos_policy_a = {"total_iops_sec": "100", + "total_bytes_sec": "1024"} + self.qos_policy_b = {"read_iops_sec": "500", + "write_iops_sec": "200"} + @ddt.data({'cluster_name': None, 'pool_name': 'rbd'}, {'cluster_name': 'volumes', 'pool_name': None}) @ddt.unpack @@ -497,11 +502,17 @@ class RBDTestCase(test.TestCase): image.update_features.assert_has_calls(calls, any_order=False) @common_mocks + @mock.patch.object(driver.RBDDriver, '_qos_specs_from_volume_type') + @mock.patch.object(driver.RBDDriver, '_supports_qos') @mock.patch.object(driver.RBDDriver, '_enable_replication') - def test_create_volume(self, mock_enable_repl): + def test_create_volume(self, mock_enable_repl, mock_qos_vers, + mock_get_qos_specs): client = self.mock_client.return_value client.__enter__.return_value = client + mock_qos_vers.return_value = True + mock_get_qos_specs.return_value = None + res = self.driver.create_volume(self.volume_a) self.assertEqual({}, res) @@ -516,6 +527,7 @@ class RBDTestCase(test.TestCase): client.__enter__.assert_called_once_with() client.__exit__.assert_called_once_with(None, None, None) mock_enable_repl.assert_not_called() + mock_qos_vers.assert_not_called() @common_mocks @mock.patch.object(driver.RBDDriver, '_enable_replication') @@ -547,6 +559,39 @@ class RBDTestCase(test.TestCase): client.__enter__.assert_called_once_with() client.__exit__.assert_called_once_with(None, None, None) + @common_mocks + @mock.patch.object(driver.RBDDriver, '_supports_qos') + @mock.patch.object(driver.RBDDriver, 'update_rbd_image_qos') + def test_create_volume_with_qos(self, mock_update_qos, mock_qos_supported): + + ctxt = context.get_admin_context() + qos = qos_specs.create(ctxt, "qos-iops-bws", self.qos_policy_a) + self.volume_a.volume_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE_ID, + qos_specs_id = qos.id) + + client = self.mock_client.return_value + client.__enter__.return_value = client + + mock_qos_supported.return_value = True + res = self.driver.create_volume(self.volume_a) + self.assertEqual({}, res) + + chunk_size = self.cfg.rbd_store_chunk_size * units.Mi + order = int(math.log(chunk_size, 2)) + args = [client.ioctx, str(self.volume_a.name), + self.volume_a.size * units.Gi, order] + kwargs = {'old_format': False, + 'features': client.features} + self.mock_rbd.RBD.return_value.create.assert_called_once_with( + *args, **kwargs) + + mock_update_qos.assert_called_once_with(self.volume_a, qos.specs) + + client.__enter__.assert_called_once_with() + client.__exit__.assert_called_once_with(None, None, None) + @common_mocks def test_manage_existing_get_size(self): with mock.patch.object(self.driver.rbd.Image(), 'size') as \ @@ -1690,14 +1735,17 @@ class RBDTestCase(test.TestCase): @ddt.data(True, False) @common_mocks + @mock.patch('cinder.volume.drivers.rbd.RBDDriver._supports_qos') @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_usage_info') @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_pool_stats') def test_update_volume_stats(self, replication_enabled, stats_mock, - usage_mock): + usage_mock, mock_qos_supported): stats_mock.return_value = (mock.sentinel.free_capacity_gb, mock.sentinel.total_capacity_gb) usage_mock.return_value = mock.sentinel.provisioned_capacity_gb + mock_qos_supported.return_value = True + expected_fsid = 'abc' expected_location_info = ('nondefault:%s:%s:%s:rbd' % (self.cfg.rbd_ceph_conf, expected_fsid, @@ -1716,7 +1764,8 @@ class RBDTestCase(test.TestCase): max_over_subscription_ratio=1.0, multiattach=True, location_info=expected_location_info, - backend_state='up') + backend_state='up', + qos_support=True) if replication_enabled: targets = [{'backend_id': 'secondary-backend'}, @@ -1735,14 +1784,21 @@ class RBDTestCase(test.TestCase): mock_get_fsid.return_value = expected_fsid actual = self.driver.get_volume_stats(True) self.assertDictEqual(expected, actual) + mock_qos_supported.assert_called_once_with() @common_mocks + @mock.patch('cinder.volume.drivers.rbd.RBDDriver._supports_qos') @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_usage_info') @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_pool_stats') - def test_update_volume_stats_exclusive_pool(self, stats_mock, usage_mock): + def test_update_volume_stats_exclusive_pool(self, stats_mock, usage_mock, + mock_qos_supported): stats_mock.return_value = (mock.sentinel.free_capacity_gb, mock.sentinel.total_capacity_gb) + # Set the version to unsupported, leading to the qos_support parameter + # in the actual output differing to the one set below in expected. + mock_qos_supported.return_value = False + expected_fsid = 'abc' expected_location_info = ('nondefault:%s:%s:%s:rbd' % (self.cfg.rbd_ceph_conf, expected_fsid, @@ -1760,7 +1816,8 @@ class RBDTestCase(test.TestCase): max_over_subscription_ratio=1.0, multiattach=True, location_info=expected_location_info, - backend_state='up') + backend_state='up', + qos_support=False) my_safe_get = MockDriverConfig(rbd_exclusive_cinder_pool=True) self.mock_object(self.driver.configuration, 'safe_get', @@ -1772,15 +1829,20 @@ class RBDTestCase(test.TestCase): self.assertDictEqual(expected, actual) usage_mock.assert_not_called() + mock_qos_supported.assert_called_once_with() @common_mocks + @mock.patch('cinder.volume.drivers.rbd.RBDDriver._supports_qos') @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_usage_info') @mock.patch('cinder.volume.drivers.rbd.RBDDriver._get_pool_stats') - def test_update_volume_stats_error(self, stats_mock, usage_mock): + def test_update_volume_stats_error(self, stats_mock, usage_mock, + mock_qos_supported): my_safe_get = MockDriverConfig(rbd_exclusive_cinder_pool=False) self.mock_object(self.driver.configuration, 'safe_get', my_safe_get) + mock_qos_supported.return_value = True + expected_fsid = 'abc' expected_location_info = ('nondefault:%s:%s:%s:rbd' % (self.cfg.rbd_ceph_conf, expected_fsid, @@ -1797,7 +1859,8 @@ class RBDTestCase(test.TestCase): max_over_subscription_ratio=1.0, thin_provisioning_support=True, location_info=expected_location_info, - backend_state='down') + backend_state='down', + qos_support=True) with mock.patch.object(self.driver, '_get_fsid') as mock_get_fsid: mock_get_fsid.return_value = expected_fsid @@ -2211,15 +2274,18 @@ class RBDTestCase(test.TestCase): self.driver.extend_volume(self.volume_a, fake_size) mock_resize.assert_called_once_with(self.volume_a, size=size) + @mock.patch.object(driver.RBDDriver, '_qos_specs_from_volume_type') + @mock.patch.object(driver.RBDDriver, '_supports_qos') @ddt.data(False, True) @common_mocks - def test_retype(self, enabled): + def test_retype(self, enabled, mock_qos_vers, mock_get_qos_specs): """Test retyping a non replicated volume. We will test on a system that doesn't have replication enabled and on one that hast it enabled. """ self.driver._is_replication_enabled = enabled + mock_qos_vers.return_value = False if enabled: expect = {'replication_status': fields.ReplicationStatus.DISABLED} else: @@ -2266,11 +2332,14 @@ class RBDTestCase(test.TestCase): {'old_replicated': True, 'new_replicated': True}) @ddt.unpack @common_mocks + @mock.patch.object(driver.RBDDriver, '_qos_specs_from_volume_type') + @mock.patch.object(driver.RBDDriver, '_supports_qos') @mock.patch.object(driver.RBDDriver, '_disable_replication', return_value={'replication': 'disabled'}) @mock.patch.object(driver.RBDDriver, '_enable_replication', return_value={'replication': 'enabled'}) - def test_retype_replicated(self, mock_disable, mock_enable, old_replicated, + def test_retype_replicated(self, mock_disable, mock_enable, mock_qos_vers, + mock_get_qos_specs, old_replicated, new_replicated): """Test retyping a non replicated volume. @@ -2285,6 +2354,9 @@ class RBDTestCase(test.TestCase): self.volume_a.volume_type = replicated_type if old_replicated else None + mock_qos_vers.return_value = False + mock_get_qos_specs.return_value = False + if new_replicated: new_type = replicated_type if old_replicated: @@ -2305,6 +2377,162 @@ class RBDTestCase(test.TestCase): None) self.assertEqual((True, update), res) + @common_mocks + @mock.patch.object(driver.RBDDriver, 'delete_rbd_image_qos_keys') + @mock.patch.object(driver.RBDDriver, 'get_rbd_image_qos') + @mock.patch.object(driver.RBDDriver, '_supports_qos') + @mock.patch.object(driver.RBDDriver, 'update_rbd_image_qos') + def test_retype_qos(self, mock_update_qos, mock_qos_supported, + mock_get_vol_qos, mock_del_vol_qos): + + ctxt = context.get_admin_context() + qos_a = qos_specs.create(ctxt, "qos-vers-a", self.qos_policy_a) + qos_b = qos_specs.create(ctxt, "qos-vers-b", self.qos_policy_b) + + # The vol_config dictionary containes supported as well as currently + # unsupported values (CNA). The latter will be marked accordingly to + # indicate the current support status. + vol_config = { + "rbd_qos_bps_burst": "0", + "rbd_qos_bps_burst_seconds": "1", # CNA + "rbd_qos_bps_limit": "1024", + "rbd_qos_iops_burst": "0", + "rbd_qos_iops_burst_seconds": "1", # CNA + "rbd_qos_iops_limit": "100", + "rbd_qos_read_bps_burst": "0", + "rbd_qos_read_bps_burst_seconds": "1", # CNA + "rbd_qos_read_bps_limit": "0", + "rbd_qos_read_iops_burst": "0", + "rbd_qos_read_iops_burst_seconds": "1", # CNA + "rbd_qos_read_iops_limit": "0", + "rbd_qos_schedule_tick_min": "50", # CNA + "rbd_qos_write_bps_burst": "0", + "rbd_qos_write_bps_burst_seconds": "1", # CNA + "rbd_qos_write_bps_limit": "0", + "rbd_qos_write_iops_burst": "0", + "rbd_qos_write_iops_burst_seconds": "1", # CNA + "rbd_qos_write_iops_limit": "0", + } + + mock_get_vol_qos.return_value = vol_config + + diff = {'encryption': {}, + 'extra_specs': {}, + 'qos_specs': {'consumer': (u'front-end', u'back-end'), + 'created_at': (123, 456), + u'total_bytes_sec': (u'1024', None), + u'total_iops_sec': (u'200', None)}} + + delete_qos = ['total_iops_sec', 'total_bytes_sec'] + + self.volume_a.volume_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE_ID, + qos_specs_id = qos_a.id) + + new_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE2_ID, + qos_specs_id = qos_b.id) + + mock_qos_supported.return_value = True + + res = self.driver.retype(ctxt, self.volume_a, new_type, diff, + None) + self.assertEqual((True, {}), res) + + assert delete_qos == [key for key in delete_qos + if key in driver.QOS_KEY_MAP] + mock_update_qos.assert_called_once_with(self.volume_a, qos_b.specs) + mock_del_vol_qos.assert_called_once_with(self.volume_a, delete_qos) + + @common_mocks + @mock.patch('cinder.volume.drivers.rbd.RBDDriver.RBDProxy') + def test__supports_qos(self, rbdproxy_mock): + rbdproxy_ver = 20 + rbdproxy_mock.return_value.version.return_value = (0, rbdproxy_ver) + + self.assertTrue(self.driver._supports_qos()) + + @common_mocks + def test__qos_specs_from_volume_type(self): + ctxt = context.get_admin_context() + qos = qos_specs.create(ctxt, "qos-vers-a", self.qos_policy_a) + self.volume_a.volume_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE_ID, + qos_specs_id = qos.id) + + self.assertEqual( + {'total_iops_sec': '100', 'total_bytes_sec': '1024'}, + self.driver._qos_specs_from_volume_type(self.volume_a.volume_type)) + + @common_mocks + def test_get_rbd_image_qos(self): + ctxt = context.get_admin_context() + qos = qos_specs.create(ctxt, "qos-vers-a", self.qos_policy_a) + self.volume_a.volume_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE_ID, + qos_specs_id = qos.id) + + rbd_image_conf = [] + for qos_key, qos_val in ( + self.volume_a.volume_type.qos_specs.specs.items()): + rbd_image_conf.append( + {'name': driver.QOS_KEY_MAP[qos_key]['ceph_key'], + 'value': int(qos_val)}) + + rbd_image = self.mock_proxy.return_value.__enter__.return_value + rbd_image.config_list.return_value = rbd_image_conf + + self.assertEqual( + {'rbd_qos_bps_limit': 1024, 'rbd_qos_iops_limit': 100}, + self.driver.get_rbd_image_qos(self.volume_a)) + + @common_mocks + def test_update_rbd_image_qos(self): + ctxt = context.get_admin_context() + qos = qos_specs.create(ctxt, "qos-vers-a", self.qos_policy_a) + self.volume_a.volume_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE_ID, + qos_specs_id = qos.id) + + rbd_image = self.mock_proxy.return_value.__enter__.return_value + + updated_specs = {"total_iops_sec": '50'} + rbd_image.config_set.return_value = qos_specs.update(ctxt, + qos.id, + updated_specs) + + self.driver.update_rbd_image_qos(self.volume_a, updated_specs) + self.assertEqual( + {'total_bytes_sec': '1024', 'total_iops_sec': '50'}, + self.volume_a.volume_type.qos_specs.specs) + + @common_mocks + def test_delete_rbd_image_qos_key(self): + ctxt = context.get_admin_context() + qos = qos_specs.create(ctxt, 'qos-vers-a', self.qos_policy_a) + self.volume_a.volume_type = fake_volume.fake_volume_type_obj( + ctxt, + id=fake.VOLUME_TYPE_ID, + qos_specs_id = qos.id) + + rbd_image = self.mock_proxy.return_value.__enter__.return_value + + keys = ['total_iops_sec'] + rbd_image.config_remove.return_value = qos_specs.delete_keys(ctxt, + qos.id, + keys) + + self.driver.delete_rbd_image_qos_keys(self.volume_a, keys) + + self.assertEqual( + {'total_bytes_sec': '1024'}, + self.volume_a.volume_type.qos_specs.specs) + @common_mocks def test_update_migrated_volume(self): client = self.mock_client.return_value diff --git a/cinder/volume/drivers/rbd.py b/cinder/volume/drivers/rbd.py index 828708bdf27..07c26c8ce6a 100644 --- a/cinder/volume/drivers/rbd.py +++ b/cinder/volume/drivers/rbd.py @@ -57,6 +57,7 @@ from cinder.objects.volume_type import VolumeType from cinder import utils from cinder.volume import configuration from cinder.volume import driver +from cinder.volume import qos_specs from cinder.volume import volume_utils LOG = logging.getLogger(__name__) @@ -140,6 +141,58 @@ CONF.register_opts(RBD_OPTS, group=configuration.SHARED_CONF_GROUP) EXTRA_SPECS_REPL_ENABLED = "replication_enabled" EXTRA_SPECS_MULTIATTACH = "multiattach" +QOS_KEY_MAP = { + 'total_iops_sec': { + 'ceph_key': 'rbd_qos_iops_limit', + 'default': 0 + }, + 'read_iops_sec': { + 'ceph_key': 'rbd_qos_read_iops_limit', + 'default': 0 + }, + 'write_iops_sec': { + 'ceph_key': 'rbd_qos_write_iops_limit', + 'default': 0 + }, + 'total_bytes_sec': { + 'ceph_key': 'rbd_qos_bps_limit', + 'default': 0 + }, + 'read_bytes_sec': { + 'ceph_key': 'rbd_qos_read_bps_limit', + 'default': 0 + }, + 'write_bytes_sec': { + 'ceph_key': 'rbd_qos_write_bps_limit', + 'default': 0 + }, + 'total_iops_sec_max': { + 'ceph_key': 'rbd_qos_bps_burst', + 'default': 0 + }, + 'read_iops_sec_max': { + 'ceph_key': 'rbd_qos_read_iops_burst', + 'default': 0 + }, + 'write_iops_sec_max': { + 'ceph_key': 'rbd_qos_write_iops_burst', + 'default': 0 + }, + 'total_bytes_sec_max': { + 'ceph_key': 'rbd_qos_bps_burst', + 'default': 0 + }, + 'read_bytes_sec_max': { + 'ceph_key': 'rbd_qos_read_bps_burst', + 'default': 0 + }, + 'write_bytes_sec_max': { + 'ceph_key': 'rbd_qos_write_bps_burst', + 'default': 0 + }} + +CEPH_QOS_SUPPORTED_VERSION = 15 + # RBD class RBDDriverException(exception.VolumeDriverException): @@ -230,9 +283,19 @@ class RADOSClient(object): class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, driver.ManageableVD, driver.ManageableSnapshotsVD, driver.BaseVD): - """Implements RADOS block device (RBD) volume commands.""" + """Implements RADOS block device (RBD) volume commands. - VERSION = '1.2.0' + + Version history: + + .. code-block:: none + + 1.3.0 - Added QoS Support + + + """ + + VERSION = '1.3.0' # ThirdPartySystems wiki page CI_WIKI_NAME = "Cinder_Jenkins" @@ -554,6 +617,9 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, ioctx.close() client.shutdown() + def _supports_qos(self): + return self.RBDProxy().version()[1] >= CEPH_QOS_SUPPORTED_VERSION + @staticmethod def _get_backup_snaps(rbd_image) -> list: """Get list of any backup snapshots that exist on this volume. @@ -688,7 +754,8 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, 'max_over_subscription_ratio': ( self.configuration.safe_get('max_over_subscription_ratio')), 'location_info': location_info, - 'backend_state': 'down' + 'backend_state': 'down', + 'qos_support': self._supports_qos(), } backend_name = self.configuration.safe_get('volume_backend_name') @@ -927,6 +994,19 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, LOG.debug('Unable to retrieve extra specs info') return False + def _qos_specs_from_volume_type(self, volume_type): + if not volume_type: + return None + + qos_specs_id = volume_type.get('qos_specs_id') + if qos_specs_id is not None: + ctxt = context.get_admin_context() + vol_qos_specs = qos_specs.get_qos_specs(ctxt, qos_specs_id) + LOG.debug('qos_specs: %s', qos_specs) + if vol_qos_specs['consumer'] in ('back-end', 'both'): + return vol_qos_specs['specs'] + return None + def _setup_volume( self, volume: Volume, @@ -941,6 +1021,16 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, had_multiattach = False volume_type = volume.volume_type + specs = self._qos_specs_from_volume_type(volume_type) + + if specs: + if self._supports_qos(): + self.update_rbd_image_qos(volume, specs) + else: + LOG.warning("Backend QOS policies for ceph not " + "supported prior to librbd version %s", + CEPH_QOS_SUPPORTED_VERSION) + want_replication = self._is_replicated_type(volume_type) want_multiattach = self._is_multiattach_type(volume_type) @@ -1446,7 +1536,47 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, new_type: VolumeType, diff: Union[dict[str, dict[str, str]], dict[str, dict], None], host: Optional[dict[str, str]]) -> tuple[bool, dict]: - """Retype from one volume type to another on the same backend.""" + """Retype from one volume type to another on the same backend. + + Returns a tuple of (diff, equal), where 'equal' is a boolean indicating + whether there is any difference, and 'diff' is a dictionary with the + following format: + + .. code-block:: default + + { + 'encryption': {}, + 'extra_specs': {}, + 'qos_specs': {'consumer': (u'front-end', u'back-end'), + u'total_bytes_sec': (None, u'2048000'), + u'total_iops_sec': (u'200', None) + {...}} + } + """ + # NOTE(rogeryu): If `diff` contains `qos_specs`, `qos_spec` must have + # the `consumer` parameter, whether or not there is a difference.] + # Remove qos keys present in RBD image that are no longer in cinder qos + # spec, new keys are added in _setup_volume. + if diff and diff.get('qos_specs') and self._supports_qos(): + specs = diff.get('qos_specs', {}) + if (specs.get('consumer') + and specs['consumer'][1] == 'front-end' + and specs['consumer'][0] != 'front-end'): + del_qos_keys = [key for key in specs.keys() + if key in QOS_KEY_MAP.keys()] + else: + del_qos_keys = [] + existing_config = self.get_rbd_image_qos(volume) + for k, v in QOS_KEY_MAP.items(): + qos_val = specs.get(k, None) + vol_val = int(existing_config.get(v['ceph_key'])) + if not qos_val: + if vol_val != v['default']: + del_qos_keys.append(k) + continue + if qos_val[1] is None and vol_val != v['default']: + del_qos_keys.append(k) + self.delete_rbd_image_qos_keys(volume, del_qos_keys) return True, self._setup_volume(volume, new_type) @staticmethod @@ -2292,3 +2422,62 @@ class RBDDriver(driver.CloneableImageVD, driver.MigrateVD, volume = objects.Volume.get_by_id(context, backup.volume_id) return (volume, False) + + @utils.retry(exception.VolumeBackendAPIException) + def get_rbd_image_qos(self, volume): + try: + with RBDVolumeProxy(self, volume.name) as rbd_image: + current = {k['name']: k['value'] + for k in rbd_image.config_list()} + return current + except Exception as e: + msg = (_("Failed to get qos specs for rbd image " + "%(rbd_image_name)s, due to " + "%(error)s.") + % {'rbd_image_name': volume.name, + 'error': e}) + raise exception.VolumeBackendAPIException( + data=msg) + + @utils.retry(exception.VolumeBackendAPIException) + def update_rbd_image_qos(self, volume, qos_specs): + try: + with RBDVolumeProxy(self, volume.name) as rbd_image: + for qos_key, qos_val in qos_specs.items(): + if qos_key in QOS_KEY_MAP: + rbd_image.config_set(QOS_KEY_MAP[qos_key]['ceph_key'], + str(qos_val)) + LOG.debug('qos_specs: %(qos_key)s successfully set to' + ' %(qos_value)s', {'qos_key': qos_key, + 'qos_value': qos_val}) + else: + LOG.warning('qos_specs: the requested qos key' + '%(qos_key)s does not exist', + {'qos_key': qos_key, + 'qos_value': qos_val}) + except Exception as e: + msg = (_('Failed to set qos spec %(qos_key)s ' + 'for rbd image %(rbd_image_name)s, ' + 'due to %(error)s.') + % {'qos_key': qos_key, + 'rbd_image_name': volume.name, + 'error': e}) + raise exception.VolumeBackendAPIException(data=msg) + + @utils.retry(exception.VolumeBackendAPIException) + def delete_rbd_image_qos_keys(self, volume, qos_keys): + try: + with RBDVolumeProxy(self, volume.name) as rbd_image: + for key in qos_keys: + rbd_image.config_remove(QOS_KEY_MAP[key]['ceph_key']) + LOG.debug('qos_specs: %(qos_key)s was ' + 'successfully unset', + {'qos_key': key}) + except Exception as e: + msg = (_("Failed to delete qos keys %(qos_key)s " + "for rbd image %(rbd_image_name)s, " + "due to %(error)s.") + % {'qos_key': key, + 'rbd_image_name': volume.name, + 'error': e}) + raise exception.VolumeBackendAPIException(data=msg) diff --git a/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst b/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst index c7a44d91b2c..e949bce3b5a 100644 --- a/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst +++ b/doc/source/configuration/block-storage/drivers/ceph-rbd-volume-driver.rst @@ -162,3 +162,67 @@ refer to the `Ceph documentation Note that with the RBD driver in cinder you need to configure the pool replication option in image mode. For instance, if your pool is named ``volumes``, the command would be: ``rbd mirror pool enable volumes image``. + +RBD QoS +~~~~~~~~~~~~~ + +Currently, the Cinder RBD driver supports the following QoS options compatible +with Ceph Octopus release and above: + +.. list-table:: + :header-rows: 1 + + * - Cinder Value + - Ceph Mapping + * - ``total_iops_sec`` + - ``rbd_qos_iops_limit`` + * - + - + * - ``read_iops_sec`` + - ``rbd_qos_read_iops_limit`` + * - + - + * - ``write_iops_sec`` + - ``rbd_qos_write_iops_limit`` + * - + - + * - ``total_bytes_sec`` + - ``rbd_qos_bps_limit`` + * - + - + * - ``read_bytes_sec`` + - ``rbd_qos_read_bps_limit`` + * - + - + * - ``write_bytes_sec`` + - ``rbd_qos_write_bps_limit`` + * - + - + * - ``total_iops_sec_max`` + - ``rbd_qos_bps_burst`` + * - + - + * - ``read_iops_sec_max`` + - ``rbd_qos_read_iops_burst`` + * - + - + * - ``write_iops_sec_max`` + - ``rbd_qos_write_iops_burst`` + * - + - + * - ``total_bytes_sec_max`` + - ``rbd_qos_bps_burst`` + * - + - + * - ``read_bytes_sec_max`` + - ``rbd_qos_read_bps_burst`` + * - + - + * - ``write_bytes_sec_max`` + - ``rbd_qos_write_bps_burst`` + * - + - + + +For more information on QoS settings you may refer to `Ceph QoS documentation +`_. diff --git a/releasenotes/notes/rbd-backend-qos-implementation-0e141b742e277d26.yaml b/releasenotes/notes/rbd-backend-qos-implementation-0e141b742e277d26.yaml new file mode 100644 index 00000000000..007ea8b1c4d --- /dev/null +++ b/releasenotes/notes/rbd-backend-qos-implementation-0e141b742e277d26.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + RBD driver: Added QoS support.