openstack/manila
Code Issues Proposed changes

[NetApp] Adding support for Adaptive QoS in NetApp driver with dhss false.

Browse Source 
Added support for Adaptive QoS policies that have been pre-created on
the storage system, with clustered ONTAP version 9.4 or higher.  To use
this feature, configure a Manila share type with the extra-spec
"netapp:adaptive_qos_policy_group" and value set to the qos policy
group on the ONTAP storage system, for example:

netapp:adaptive_qos_policy_group=platform3

Note that a cluster scoped account must be used in the driver
configuration in order to use QoS in clustered ONTAP.  Other notes:

-This only works for backends without share server management.
-This does not work for share replicas or share migration.

Partially-Implements: bp netapp-adaptive-qos-support

Change-Id: I3cc1d2fa2a8380ca925538cab5a3414ac2141d70
This commit is contained in:
Michael Arndt 2020-04-13 14:25:23 +00:00 committed by Douglas Viroel
parent 4bcf21eaf1
commit 2f0981602b
6 changed files with 186 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
manila/share/drivers/netapp/dataontap/client/client_cmode.py
View File

@ -85,6 +85,7 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
supported=ontapi_1_30)
self.features.add_feature('FLEXVOL_ENCRYPTION', supported=ontapi_1_110)
self.features.add_feature('SVM_DR', supported=ontapi_1_140)
self.features.add_feature('ADAPTIVE_QOS', supported=ontapi_1_140)
self.features.add_feature('TRANSFER_LIMIT_NFS_CONFIG',
supported=ontapi_1_140)
self.features.add_feature('CIFS_DC_ADD_SKIP_CHECK',
@ -1646,7 +1647,7 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
language=None, dedup_enabled=False,
compression_enabled=False, max_files=None,
snapshot_reserve=None, volume_type='rw',
qos_policy_group=None,
qos_policy_group=None, adaptive_qos_policy_group=None,
encrypt=False, **options):
"""Creates a volume."""
api_args = {
@ -1668,6 +1669,9 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
snapshot_reserve)
if qos_policy_group is not None:
api_args['qos-policy-group-name'] = qos_policy_group
if adaptive_qos_policy_group is not None:
api_args['qos-adaptive-policy-group-name'] = (
adaptive_qos_policy_group)
if encrypt is True:
if not self.features.FLEXVOL_ENCRYPTION:
@ -2323,7 +2327,8 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
@na_utils.trace
def create_volume_clone(self, volume_name, parent_volume_name,
parent_snapshot_name=None, split=False,
qos_policy_group=None, **options):
qos_policy_group=None,
adaptive_qos_policy_group=None, **options):
"""Clones a volume."""
api_args = {
'volume': volume_name,
@ -2340,6 +2345,10 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
if split:
self.split_volume_clone(volume_name)
if adaptive_qos_policy_group is not None:
self.set_qos_adaptive_policy_group_for_volume(
volume_name, adaptive_qos_policy_group)
@na_utils.trace
def split_volume_clone(self, volume_name):
"""Begins splitting a clone from its parent."""
@ -3012,6 +3021,31 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
}
self.send_request('volume-modify-iter', api_args)
@na_utils.trace
def set_qos_adaptive_policy_group_for_volume(self, volume_name,
qos_policy_group_name):
if not self.features.ADAPTIVE_QOS:
msg = 'Adaptive QoS not supported on this backend ONTAP version.'
raise exception.NetAppException(msg)
api_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': {
'name': volume_name,
},
},
},
'attributes': {
'volume-attributes': {
'volume-qos-attributes': {
'adaptive-policy-group-name': qos_policy_group_name,
},
},
},
}
self.send_request('volume-modify-iter', api_args)
@na_utils.trace
def get_nfs_export_policy_for_volume(self, volume_name):
"""Get the name of the export policy for a volume."""

15
manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py
View File

@ -84,6 +84,7 @@ class NetAppCmodeFileStorageLibrary(object):
'netapp:snapshot_policy': 'snapshot_policy',
'netapp:language': 'language',
'netapp:max_files': 'max_files',
'netapp:adaptive_qos_policy_group': 'adaptive_qos_policy_group',
}
# Maps standard extra spec keys to legacy NetApp keys
@ -1039,6 +1040,20 @@ class NetAppCmodeFileStorageLibrary(object):
self._check_extra_specs_validity(share, extra_specs)
provisioning_options = self._get_provisioning_options(extra_specs)
qos_specs = self._get_normalized_qos_specs(extra_specs)
if (provisioning_options.get('adaptive_qos_policy_group') is not None
and qos_specs):
msg = _('Share cannot be provisioned with both qos_specs '
'%(qos_specs_string)s and adaptive_qos_policy_group '
'%(adaptive_qos_policy_group)s.')
qos_specs_string = ""
for key in qos_specs:
qos_specs_string += key + "=" + str(qos_specs[key]) + " "
msg_args = {
'adaptive_qos_policy_group':
provisioning_options['adaptive_qos_policy_group'],
'qos_specs_string': qos_specs_string
}
raise exception.NetAppException(msg % msg_args)
if qos_specs and not replica:
qos_policy_group = self._create_qos_policy_group(
share, vserver, qos_specs, vserver_client)

78
manila/tests/share/drivers/netapp/dataontap/client/test_client_cmode.py
View File

@ -2839,8 +2839,16 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.client.send_request.assert_called_once_with('volume-create',
volume_create_args)
@ddt.data(None, fake.QOS_POLICY_GROUP_NAME)
def test_create_volume_with_extra_specs(self, qos_policy_group_name):
@ddt.data({'qos_policy_group_name': None,
'adaptive_policy_group_name': None},
{'qos_policy_group_name': fake.QOS_POLICY_GROUP_NAME,
'adaptive_policy_group_name': None},
{'qos_policy_group_name': None,
'adaptive_policy_group_name': fake.QOS_POLICY_GROUP_NAME},
)
@ddt.unpack
def test_create_volume_with_extra_specs(self, qos_policy_group_name,
adaptive_policy_group_name):
self.mock_object(self.client, 'set_volume_max_files')
self.mock_object(self.client, 'enable_dedup')
@ -2856,7 +2864,8 @@ class NetAppClientCmodeTestCase(test.TestCase):
thin_provisioned=True, language='en-US',
snapshot_policy='default', dedup_enabled=True,
compression_enabled=True, max_files=5000, snapshot_reserve=15,
qos_policy_group=qos_policy_group_name)
qos_policy_group=qos_policy_group_name,
adaptive_qos_policy_group=adaptive_policy_group_name)
volume_create_args = {
'containing-aggr-name': fake.SHARE_AGGREGATE_NAME,
@ -2873,6 +2882,9 @@ class NetAppClientCmodeTestCase(test.TestCase):
if qos_policy_group_name:
volume_create_args.update(
{'qos-policy-group-name': qos_policy_group_name})
if adaptive_policy_group_name:
volume_create_args.update(
{'qos-adaptive-policy-group-name': adaptive_policy_group_name})
self.client.send_request.assert_called_with('volume-create',
volume_create_args)
@ -3961,16 +3973,29 @@ class NetAppClientCmodeTestCase(test.TestCase):
self.assertIsNone(result)
@ddt.data(None, fake.QOS_POLICY_GROUP_NAME)
def test_create_volume_clone(self, qos_policy_group_name):
@ddt.data({'qos_policy_group_name': None,
'adaptive_qos_policy_group_name': None},
{'qos_policy_group_name': fake.QOS_POLICY_GROUP_NAME,
'adaptive_qos_policy_group_name': None},
{'qos_policy_group_name': None,
'adaptive_qos_policy_group_name': fake.QOS_POLICY_GROUP_NAME},
)
@ddt.unpack
def test_create_volume_clone(self, qos_policy_group_name,
adaptive_qos_policy_group_name):
self.mock_object(self.client, 'send_request')
self.mock_object(self.client, 'split_volume_clone')
set_qos_adapt_mock = self.mock_object(
self.client,
'set_qos_adaptive_policy_group_for_volume')
self.client.create_volume_clone(fake.SHARE_NAME,
fake.PARENT_SHARE_NAME,
fake.PARENT_SNAPSHOT_NAME,
qos_policy_group=qos_policy_group_name)
self.client.create_volume_clone(
fake.SHARE_NAME,
fake.PARENT_SHARE_NAME,
fake.PARENT_SNAPSHOT_NAME,
qos_policy_group=qos_policy_group_name,
adaptive_qos_policy_group=adaptive_qos_policy_group_name)
volume_clone_create_args = {
'volume': fake.SHARE_NAME,
@ -3982,7 +4007,10 @@ class NetAppClientCmodeTestCase(test.TestCase):
if qos_policy_group_name:
volume_clone_create_args.update(
{'qos-policy-group-name': fake.QOS_POLICY_GROUP_NAME})
if adaptive_qos_policy_group_name:
set_qos_adapt_mock.assert_called_once_with(
fake.SHARE_NAME, fake.QOS_POLICY_GROUP_NAME
)
self.client.send_request.assert_has_calls([
mock.call('volume-clone-create', volume_clone_create_args)])
self.assertFalse(self.client.split_volume_clone.called)
@ -7092,6 +7120,36 @@ class NetAppClientCmodeTestCase(test.TestCase):
'volume-get-iter', expected_api_args)
self.assertEqual(expected_snapshot_name, result)
def test_set_qos_adaptive_policy_group_for_volume(self):
self.client.features.add_feature('ADAPTIVE_QOS')
self.mock_object(self.client, 'send_request')
self.client.set_qos_adaptive_policy_group_for_volume(
fake.SHARE_NAME,
fake.QOS_POLICY_GROUP_NAME)
volume_modify_iter_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': {
'name': fake.SHARE_NAME,
},
},
},
'attributes': {
'volume-attributes': {
'volume-qos-attributes': {
'adaptive-policy-group-name':
fake.QOS_POLICY_GROUP_NAME,
},
},
},
}
self.client.send_request.assert_called_once_with(
'volume-modify-iter', volume_modify_iter_args)
def test_get_nfs_config(self):
api_args = {
'query': {

43
manila/tests/share/drivers/netapp/dataontap/cluster_mode/test_lib_base.py
View File

@ -1278,7 +1278,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
fake.POOL_NAME, fake.SHARE_NAME, fake.SHARE['size'],
thin_provisioned=True, snapshot_policy='default',
language='en-US', dedup_enabled=True, split=True, encrypt=False,
compression_enabled=False, max_files=5000, snapshot_reserve=8)
compression_enabled=False, max_files=5000, snapshot_reserve=8,
adaptive_qos_policy_group=None)
if hide_snapdir:
vserver_client.set_volume_snapdir_access.assert_called_once_with(
@ -1316,7 +1317,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
thin_provisioned=True, snapshot_policy='default',
language='en-US', dedup_enabled=True, split=True,
compression_enabled=False, max_files=5000, encrypt=False,
snapshot_reserve=8, volume_type='dp')
snapshot_reserve=8, volume_type='dp',
adaptive_qos_policy_group=None)
def test_allocate_container_no_pool_name(self):
self.mock_object(self.library, '_get_backend_share_name', mock.Mock(
@ -1448,11 +1450,45 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
mock_get_provisioning_options.assert_called_once_with(extra_specs)
mock_get_normalized_qos_specs.assert_called_once_with(extra_specs)
def test_get_provisioning_options_for_share_qos_conflict(self):
vserver_client = mock.Mock()
extra_specs = fake.EXTRA_SPEC_WITH_QOS
mock_get_extra_specs_from_share = self.mock_object(
share_types, 'get_extra_specs_from_share',
mock.Mock(return_value=extra_specs))
mock_remap_standard_boolean_extra_specs = self.mock_object(
self.library, '_remap_standard_boolean_extra_specs',
mock.Mock(return_value=extra_specs))
mock_check_extra_specs_validity = self.mock_object(
self.library, '_check_extra_specs_validity')
mock_get_provisioning_options = self.mock_object(
self.library, '_get_provisioning_options',
mock.Mock(return_value=fake.PROVISIONING_OPTS_WITH_ADAPT_QOS))
mock_get_normalized_qos_specs = self.mock_object(
self.library, '_get_normalized_qos_specs',
mock.Mock(return_value={fake.QOS_NORMALIZED_SPEC: 3000}))
self.assertRaises(exception.NetAppException,
self.library._get_provisioning_options_for_share,
fake.SHARE_INSTANCE, fake.VSERVER1,
vserver_client=vserver_client,
replica=False)
mock_get_extra_specs_from_share.assert_called_once_with(
fake.SHARE_INSTANCE)
mock_remap_standard_boolean_extra_specs.assert_called_once_with(
extra_specs)
mock_check_extra_specs_validity.assert_called_once_with(
fake.SHARE_INSTANCE, extra_specs)
mock_get_provisioning_options.assert_called_once_with(extra_specs)
mock_get_normalized_qos_specs.assert_called_once_with(extra_specs)
def test_get_provisioning_options_implicit_false(self):
result = self.library._get_provisioning_options(
fake.EMPTY_EXTRA_SPEC)
expected = {
'adaptive_qos_policy_group': None,
'language': None,
'max_files': None,
'snapshot_policy': None,
@ -1662,7 +1698,8 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
share_name, parent_share_name, parent_snapshot_name,
thin_provisioned=True, snapshot_policy='default',
language='en-US', dedup_enabled=True, split=expected_split_op,
encrypt=False, compression_enabled=False, max_files=5000)
encrypt=False, compression_enabled=False, max_files=5000,
adaptive_qos_policy_group=None)
if size > original_snapshot_size:
vserver_client.set_volume_size.assert_called_once_with(
share_name, size)

11
manila/tests/share/drivers/netapp/dataontap/fakes.py
View File

@ -191,6 +191,7 @@ EXTRA_SPEC = {
'netapp_flexvol_encryption': 'true',
'netapp:tcp_max_xfer_size': 100,
'netapp:udp_max_xfer_size': 100,
'netapp:adaptive_qos_policy_group': None,
}
NFS_CONFIG_DEFAULT = {
@ -271,12 +272,17 @@ PROVISIONING_OPTIONS = {
'split': True,
'encrypt': False,
'hide_snapdir': False,
'adaptive_qos_policy_group': None,
}
PROVISIONING_OPTIONS_WITH_QOS = copy.deepcopy(PROVISIONING_OPTIONS)
PROVISIONING_OPTIONS_WITH_QOS.update(
{'qos_policy_group': QOS_POLICY_GROUP_NAME})
PROVISIONING_OPTS_WITH_ADAPT_QOS = copy.deepcopy(PROVISIONING_OPTIONS)
PROVISIONING_OPTS_WITH_ADAPT_QOS.update(
{'adaptive_qos_policy_group': QOS_POLICY_GROUP_NAME})
PROVISIONING_OPTIONS_BOOLEAN = {
'thin_provisioned': True,
'dedup_enabled': False,
@ -300,18 +306,21 @@ PROVISIONING_OPTIONS_STRING = {
'snapshot_policy': 'default',
'language': 'en-US',
'max_files': 5000,
'adaptive_qos_policy_group': None,
}
PROVISIONING_OPTIONS_STRING_MISSING_SPECS = {
'snapshot_policy': 'default',
'language': 'en-US',
'max_files': None,
'adaptive_qos_policy_group': None,
}
PROVISIONING_OPTIONS_STRING_DEFAULT = {
'snapshot_policy': None,
'language': None,
'max_files': None,
'adaptive_qos_policy_group': None,
}
SHORT_BOOLEAN_EXTRA_SPEC = {
@ -322,11 +331,13 @@ STRING_EXTRA_SPEC = {
'netapp:snapshot_policy': 'default',
'netapp:language': 'en-US',
'netapp:max_files': 5000,
'netapp:adaptive_qos_policy_group': None,
}
SHORT_STRING_EXTRA_SPEC = {
'netapp:snapshot_policy': 'default',
'netapp:language': 'en-US',
'netapp:adaptive_qos_policy_group': None,
}
INVALID_EXTRA_SPEC = {

16
releasenotes/notes/netapp-add-support-for-adaptive-qos-d036238e7f29cf75.yaml Normal file
View File

@ -0,0 +1,16 @@
---
features:
- |
Added support for Adaptive QoS policies that have been pre-created on
the storage system, with clustered ONTAP version 9.4 or higher. To use
this feature, configure a Manila share type with the extra-spec
"netapp:adaptive_qos_policy_group" and value set to the qos policy
group on the ONTAP storage system, for example:
netapp:adaptive_qos_policy_group=platform3
Note that a cluster scoped account must be used in the driver
configuration in order to use QoS in clustered ONTAP. Other notes:
- This only works for backends without share server management.
- This does not work for share replicas or share migration.