openstack
/
cinder
Code Issues Proposed changes

NetApp: Decouple capacity volume stats collection 

The NetApp cDOT drivers keep their pool capacities in the
data structures used for other pool attributes that support
various volume type extra specs.  But the capacity info is
fast-changing, while the other pool attributes rarely or never
change.  A subsequent patch will simplify the pool attribute
collection code by refreshing itself on a slow cadence using
loopingcall.  This patch splits the pool capacity collection
so that it may be called independently on each call to
_update_volume_stats.

Partially implements: blueprint replace-netapp-cdot-ssc-module
Change-Id: I526a611cf6459417c63366f9fa21f2a803727910
This commit is contained in:
Clinton Knight 2016-04-14 01:54:01 -04:00
parent 8091e9f737
commit d0f7842110
14 changed files with 163 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cinder/tests/unit/test_netapp_ssc.py
View File

@ -581,8 +581,6 @@ class SscUtilsTestCase(test.TestCase):
<type>rw</type>
</volume-id-attributes>
<volume-space-attributes>
<size-available>214748364</size-available>
<size-total>224748364</size-total>
<space-guarantee-enabled>enabled</space-guarantee-enabled>
<space-guarantee>file</space-guarantee>
</volume-space-attributes>
@ -607,8 +605,6 @@ class SscUtilsTestCase(test.TestCase):
<type>rw</type>
</volume-id-attributes>
<volume-space-attributes>
<size-available>14748364</size-available>
<size-total>24748364</size-total>
<space-guarantee-enabled>enabled
</space-guarantee-enabled>
<space-guarantee>volume</space-guarantee>
@ -634,8 +630,6 @@ class SscUtilsTestCase(test.TestCase):
<type>rw</type>
</volume-id-attributes>
<volume-space-attributes>
<size-available>14748364</size-available>
<size-total>24748364</size-total>
<space-guarantee-enabled>enabled
</space-guarantee-enabled>
<space-guarantee>volume</space-guarantee>
@ -683,7 +677,7 @@ class SscUtilsTestCase(test.TestCase):
return_value=[netapp_api.NaElement(body)]))
vols = ssc_cmode.query_cluster_vols_for_ssc(na_server, 'Openstack')
self.assertEqual(2, len(vols))
self.assertEqual(3, len(vols))
for vol in vols:
if vol.id['name'] != 'iscsi' or vol.id['name'] != 'nfsvol':
pass

32
cinder/tests/unit/volume/drivers/netapp/dataontap/client/fakes.py
View File

@ -104,6 +104,13 @@ NO_RECORDS_RESPONSE = etree.XML("""
</results>
""")
INVALID_GET_ITER_RESPONSE_NO_RECORDS = etree.XML("""
<results status="passed">
<attributes-list/>
<next-tag>fake_tag</next-tag>
</results>
""")
GET_OPERATIONAL_NETWORK_INTERFACE_ADDRESSES_RESPONSE = etree.XML("""
<results status="passed">
<num-records>2</num-records>
@ -280,12 +287,6 @@ SNAPSHOT_NOT_PRESENT_7MODE = etree.XML("""
</results>
""" % {'vol_name': fake.SNAPSHOT['volume_id']})
NO_RECORDS_RESPONSE = etree.XML("""
<results status="passed">
<num-records>0</num-records>
</results>
""")
NODE_NAME = 'fake_node1'
NODE_NAMES = ('fake_node1', 'fake_node2')
VOLUME_AGGREGATE_NAME = 'fake_aggr1'
@ -608,6 +609,25 @@ AGGR_GET_NODE_RESPONSE = etree.XML("""
'node': NODE_NAME,
})
VOLUME_SIZE_TOTAL = 19922944
VOLUME_SIZE_AVAILABLE = 19791872
VOLUME_GET_ITER_RESPONSE = etree.XML("""
<results status="passed">
<num-records>1</num-records>
<attributes-list>
<volume-attributes>
<volume-space-attributes>
<size-available>%(available_size)s</size-available>
<size-total>%(total_size)s</size-total>
</volume-space-attributes>
</volume-attributes>
</attributes-list>
</results>
""" % {
'available_size': VOLUME_SIZE_AVAILABLE,
'total_size': VOLUME_SIZE_TOTAL,
})
PERF_OBJECT_COUNTER_TOTAL_CP_MSECS_LABELS = [
'SETUP', 'PRE_P0', 'P0_SNAP_DEL', 'P1_CLEAN', 'P1_QUOTA', 'IPU_DISK_ADD',
'P2V_INOFILE', 'P2V_INO_PUB', 'P2V_INO_PRI', 'P2V_FSINFO', 'P2V_DLOG1',

10
cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_7mode.py
View File

@ -670,11 +670,13 @@ class NetApp7modeClientTestCase(test.TestCase):
'available_bytes': expected_available_bytes}))
self.connection.invoke_successfully.return_value = response
total_bytes, available_bytes = (
self.client.get_flexvol_capacity(fake_flexvol_path))
result = self.client.get_flexvol_capacity(fake_flexvol_path)
self.assertEqual(expected_total_bytes, total_bytes)
self.assertEqual(expected_available_bytes, available_bytes)
expected = {
'size-total': expected_total_bytes,
'size-available': expected_available_bytes,
}
self.assertEqual(expected, result)
def test_get_performance_instance_names(self):

86
cinder/tests/unit/volume/drivers/netapp/dataontap/client/test_client_cmode.py
View File

@ -17,6 +17,7 @@
import uuid
import ddt
from lxml import etree
import mock
import paramiko
@ -41,6 +42,7 @@ CONNECTION_INFO = {'hostname': 'hostname',
'vserver': 'fake_vserver'}
@ddt.ddt
class NetAppCmodeClientTestCase(test.TestCase):
def setUp(self):
@ -81,6 +83,26 @@ class NetAppCmodeClientTestCase(test.TestCase):
self.assertFalse(result)
@ddt.data((fake_client.AGGR_GET_ITER_RESPONSE, 2),
(fake_client.NO_RECORDS_RESPONSE, 0))
@ddt.unpack
def test_get_record_count(self, response, expected):
api_response = netapp_api.NaElement(response)
result = self.client._get_record_count(api_response)
self.assertEqual(expected, result)
def test_get_records_count_invalid(self):
api_response = netapp_api.NaElement(
fake_client.INVALID_GET_ITER_RESPONSE_NO_RECORDS)
self.assertRaises(exception.NetAppDriverException,
self.client._get_record_count,
api_response)
def test_get_iscsi_target_details_no_targets(self):
response = netapp_api.NaElement(
etree.XML("""<results status="passed">
@ -936,31 +958,53 @@ class NetAppCmodeClientTestCase(test.TestCase):
self.assertEqual(expected_result, address_list)
def test_get_flexvol_capacity(self):
expected_total_size = 1000
expected_available_size = 750
fake_flexvol_path = '/fake/vol'
api_response = netapp_api.NaElement(
etree.XML("""
<results status="passed">
<attributes-list>
<volume-attributes>
<volume-space-attributes>
<size-available>%(available_size)s</size-available>
<size-total>%(total_size)s</size-total>
</volume-space-attributes>
</volume-attributes>
</attributes-list>
</results>""" % {'available_size': expected_available_size,
'total_size': expected_total_size}))
@ddt.data({'flexvol_path': '/fake/vol'},
{'flexvol_name': 'fake_volume'},
{'flexvol_path': '/fake/vol', 'flexvol_name': 'fake_volume'})
def test_get_flexvol_capacity(self, kwargs):
api_response = netapp_api.NaElement(
fake_client.VOLUME_GET_ITER_RESPONSE)
self.mock_send_request.return_value = api_response
total_size, available_size = (
self.client.get_flexvol_capacity(fake_flexvol_path))
capacity = self.client.get_flexvol_capacity(**kwargs)
self.assertEqual(expected_total_size, total_size)
self.assertEqual(expected_available_size, available_size)
volume_id_attributes = {}
if 'flexvol_path' in kwargs:
volume_id_attributes['junction-path'] = kwargs['flexvol_path']
if 'flexvol_name' in kwargs:
volume_id_attributes['name'] = kwargs['flexvol_name']
volume_get_iter_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': volume_id_attributes,
}
},
'desired-attributes': {
'volume-attributes': {
'volume-space-attributes': {
'size-available': None,
'size-total': None,
}
}
},
}
self.mock_send_request.assert_called_once_with(
'volume-get-iter', volume_get_iter_args)
self.assertEqual(fake_client.VOLUME_SIZE_TOTAL, capacity['size-total'])
self.assertEqual(fake_client.VOLUME_SIZE_AVAILABLE,
capacity['size-available'])
def test_get_flexvol_capacity_not_found(self):
self.mock_send_request.return_value = netapp_api.NaElement(
fake_client.NO_RECORDS_RESPONSE)
self.assertRaises(exception.NetAppDriverException,
self.client.get_flexvol_capacity,
flexvol_path='fake_path')
def test_get_aggregates(self):

3
cinder/tests/unit/volume/drivers/netapp/dataontap/fakes.py
View File

@ -169,6 +169,7 @@ MAX_OVER_SUBSCRIPTION_RATIO = 19.0
TOTAL_BYTES = 4797892092432
AVAILABLE_BYTES = 13479932478
CAPACITY_VALUES = (TOTAL_BYTES, AVAILABLE_BYTES)
CAPACITIES = {'size-total': TOTAL_BYTES, 'size-available': AVAILABLE_BYTES}
IGROUP1 = {'initiator-group-os-type': 'linux',
'initiator-group-type': 'fcp',
@ -279,8 +280,6 @@ FAKE_CMODE_VOL1 = ssc_cmode.NetAppVolume(name='open123', vserver='openstack')
FAKE_CMODE_VOL1.state['vserver_root'] = False
FAKE_CMODE_VOL1.state['status'] = 'online'
FAKE_CMODE_VOL1.state['junction_active'] = True
FAKE_CMODE_VOL1.space['size_avl_bytes'] = '4000000000'
FAKE_CMODE_VOL1.space['size_total_bytes'] = '5000000000'
FAKE_CMODE_VOL1.space['space-guarantee-enabled'] = False
FAKE_CMODE_VOL1.space['space-guarantee'] = 'file'
FAKE_CMODE_VOL1.space['thin_provisioned'] = True

16
cinder/tests/unit/volume/drivers/netapp/dataontap/test_block_cmode.py
View File

@ -338,9 +338,7 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
'raid_type': 'raiddp'
}
test_volume.space = {
'size_total_bytes': '10737418240',
'space-guarantee': 'file',
'size_avl_bytes': '2147483648',
'space-guarantee-enabled': False,
'thin_provisioned': False
}
@ -366,6 +364,13 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
netapp_lun_space_reservation)
self.library.perf_library.get_node_utilization_for_pool = (
mock.Mock(return_value=30.0))
mock_capacities = {
'size-total': 10737418240.0,
'size-available': 2147483648.0,
}
self.mock_object(
self.zapi_client, 'get_flexvol_capacity',
mock.Mock(return_value=mock_capacities))
netapp_thin = 'true' if thin else 'false'
netapp_thick = 'false' if thin else 'true'
@ -677,6 +682,13 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
mock.Mock(return_value=[fake.FAKE_CMODE_VOL1]))
self.library.perf_library.get_node_utilization_for_pool = (
mock.Mock(return_value=30.0))
mock_capacities = {
'size-total': 5000000000.0,
'size-available': 4000000000.0,
}
self.mock_object(
self.zapi_client, 'get_flexvol_capacity',
mock.Mock(return_value=mock_capacities))
pools = self.library._get_pool_stats(filter_function='filter',
goodness_function='goodness')

8
cinder/tests/unit/volume/drivers/netapp/dataontap/test_nfs_base.py
View File

@ -86,25 +86,25 @@ class NetAppNfsDriverTestCase(test.TestCase):
expected = fake.CAPACITY_VALUES
self.driver.zapi_client = mock.Mock()
get_capacity = self.driver.zapi_client.get_flexvol_capacity
get_capacity.return_value = fake.CAPACITY_VALUES
get_capacity.return_value = fake.CAPACITIES
result = self.driver._get_capacity_info(fake.NFS_SHARE_IPV4)
self.assertEqual(expected, result)
get_capacity.assert_has_calls([
mock.call(fake.EXPORT_PATH)])
mock.call(flexvol_path=fake.EXPORT_PATH)])
def test_get_capacity_info_ipv6_share(self):
expected = fake.CAPACITY_VALUES
self.driver.zapi_client = mock.Mock()
get_capacity = self.driver.zapi_client.get_flexvol_capacity
get_capacity.return_value = fake.CAPACITY_VALUES
get_capacity.return_value = fake.CAPACITIES
result = self.driver._get_capacity_info(fake.NFS_SHARE_IPV6)
self.assertEqual(expected, result)
get_capacity.assert_has_calls([
mock.call(fake.EXPORT_PATH)])
mock.call(flexvol_path=fake.EXPORT_PATH)])
def test_create_volume(self):
self.mock_object(self.driver, '_ensure_shares_mounted')

2
cinder/tests/unit/volume/drivers/netapp/test_utils.py
View File

@ -104,10 +104,12 @@ class NetAppDriverUtilsTestCase(test.TestCase):
self.assertEqual('new_fake_value', fake_object.fake_attr)
def test_round_down(self):
self.assertAlmostEqual(na_utils.round_down(5.567), 5.56)
self.assertAlmostEqual(na_utils.round_down(5.567, '0.00'), 5.56)
self.assertAlmostEqual(na_utils.round_down(5.567, '0.0'), 5.5)
self.assertAlmostEqual(na_utils.round_down(5.567, '0'), 5)
self.assertAlmostEqual(na_utils.round_down(0, '0.00'), 0)
self.assertAlmostEqual(na_utils.round_down(-5.567), -5.56)
self.assertAlmostEqual(na_utils.round_down(-5.567, '0.00'), -5.56)
self.assertAlmostEqual(na_utils.round_down(-5.567, '0.0'), -5.5)
self.assertAlmostEqual(na_utils.round_down(-5.567, '0'), -5)

15
cinder/volume/drivers/netapp/dataontap/block_cmode.py
View File

@ -228,14 +228,15 @@ class NetAppBlockStorageCmodeLibrary(block_base.NetAppBlockStorageLibrary):
pool['max_over_subscription_ratio'] = (
self.max_over_subscription_ratio)
# convert sizes to GB
total = float(vol.space['size_total_bytes'])
total /= units.Gi
pool['total_capacity_gb'] = na_utils.round_down(total, '0.01')
# Get capacity info and convert to GB
capacity = self.zapi_client.get_flexvol_capacity(
flexvol_name=pool_name)
free = float(vol.space['size_avl_bytes'])
free /= units.Gi
pool['free_capacity_gb'] = na_utils.round_down(free, '0.01')
size_total_gb = capacity['size-total'] / units.Gi
pool['total_capacity_gb'] = na_utils.round_down(size_total_gb)
size_available_gb = capacity['size-available'] / units.Gi
pool['free_capacity_gb'] = na_utils.round_down(size_available_gb)
pool['provisioned_capacity_gb'] = (round(
pool['total_capacity_gb'] - pool['free_capacity_gb'], 2))

10
cinder/volume/drivers/netapp/dataontap/client/client_7mode.py
View File

@ -465,12 +465,14 @@ class Client(client_base.Client):
flexvol_info_list = result.get_child_by_name('volumes')
flexvol_info = flexvol_info_list.get_children()[0]
total_bytes = float(
flexvol_info.get_child_content('size-total'))
available_bytes = float(
size_total = float(flexvol_info.get_child_content('size-total'))
size_available = float(
flexvol_info.get_child_content('size-available'))
return total_bytes, available_bytes
return {
'size-total': size_total,
'size-available': size_available,
}
def get_performance_instance_names(self, object_name):
"""Get names of performance instances for a node."""

28
cinder/volume/drivers/netapp/dataontap/client/client_cmode.py
View File

@ -72,6 +72,13 @@ class Client(client_base.Client):
num_records = api_result_element.get_child_content('num-records')
return bool(num_records and '0' != num_records)
def _get_record_count(self, api_result_element):
try:
return int(api_result_element.get_child_content('num-records'))
except TypeError:
msg = _('Missing record count for NetApp iterator API invocation.')
raise exception.NetAppDriverException(msg)
def set_vserver(self, vserver):
self.connection.set_vserver(vserver)
@ -691,15 +698,19 @@ class Client(client_base.Client):
return [lif_info.get_child_content('address') for lif_info in
lif_info_list.get_children()]
def get_flexvol_capacity(self, flexvol_path):
def get_flexvol_capacity(self, flexvol_path=None, flexvol_name=None):
"""Gets total capacity and free capacity, in bytes, of the flexvol."""
volume_id_attributes = {}
if flexvol_path:
volume_id_attributes['junction-path'] = flexvol_path
if flexvol_name:
volume_id_attributes['name'] = flexvol_name
api_args = {
'query': {
'volume-attributes': {
'volume-id-attributes': {
'junction-path': flexvol_path
}
'volume-id-attributes': volume_id_attributes,
}
},
'desired-attributes': {
@ -713,6 +724,10 @@ class Client(client_base.Client):
}
result = self.send_request('volume-get-iter', api_args)
if self._get_record_count(result) != 1:
msg = _('Volume %s not found.')
msg_args = flexvol_path or flexvol_name
raise exception.NetAppDriverException(msg % msg_args)
attributes_list = result.get_child_by_name('attributes-list')
volume_attributes = attributes_list.get_child_by_name(
@ -725,7 +740,10 @@ class Client(client_base.Client):
size_total = float(
volume_space_attributes.get_child_content('size-total'))
return size_total, size_available
return {
'size-total': size_total,
'size-available': size_available,
}
@utils.trace_method
def delete_file(self, path_to_file):

8
cinder/volume/drivers/netapp/dataontap/nfs_base.py
View File

@ -800,9 +800,9 @@ class NetAppNfsDriver(driver.ManageableVD,
self.max_over_subscription_ratio)
total_size, total_available = self._get_capacity_info(nfs_share)
capacity['total_capacity_gb'] = na_utils.round_down(
total_size / units.Gi, '0.01')
total_size / units.Gi)
capacity['free_capacity_gb'] = na_utils.round_down(
total_available / units.Gi, '0.01')
total_available / units.Gi)
capacity['provisioned_capacity_gb'] = (round(
capacity['total_capacity_gb'] - capacity['free_capacity_gb'], 2))
@ -811,7 +811,9 @@ class NetAppNfsDriver(driver.ManageableVD,
def _get_capacity_info(self, nfs_share):
"""Get total capacity and free capacity in bytes for an nfs share."""
export_path = nfs_share.rsplit(':', 1)[1]
return self.zapi_client.get_flexvol_capacity(export_path)
capacity = self.zapi_client.get_flexvol_capacity(
flexvol_path=export_path)
return capacity['size-total'], capacity['size-available']
def _check_volume_type(self, volume, share, file_name, extra_specs):
"""Match volume type for share file."""

25
cinder/volume/drivers/netapp/dataontap/ssc_cmode.py
View File

@ -47,8 +47,7 @@ class NetAppVolume(object):
state - status, vserver_root, cluster_volume,
inconsistent, invalid, junction_active
qos - qos_policy_group
space - space-guarantee-enabled, space-guarantee,
thin_provisioned, size_avl_bytes, size_total_bytes
space - space-guarantee-enabled, space-guarantee, thin_provisioned
mirror - mirrored i.e. dp mirror
export - path
"""
@ -74,23 +73,6 @@ class NetAppVolume(object):
"""Computes hash for the object."""
return hash(self.id['name'])
def __cmp__(self, other):
"""Implements comparison logic for volumes."""
self_size_avl = self.space.get('size_avl_bytes')
other_size_avl = other.space.get('size_avl_bytes')
if self_size_avl is None and other_size_avl is not None:
return -1
elif self_size_avl is not None and other_size_avl is None:
return 1
elif self_size_avl is None and other_size_avl is None:
return 0
elif int(self_size_avl) < int(other_size_avl):
return -1
elif int(self_size_avl) > int(other_size_avl):
return 1
else:
return 0
def __str__(self):
"""Returns human readable form for object."""
vol_str = "NetApp Volume id: %s, aggr: %s,"\
@ -231,11 +213,6 @@ def create_vol_list(vol_attrs):
# aggr attributes mandatory.
vol.aggr['name'] =\
v['volume-id-attributes']['containing-aggregate-name']
# space attributes mandatory.
vol.space['size_avl_bytes'] =\
v['volume-space-attributes']['size-available']
vol.space['size_total_bytes'] =\
v['volume-space-attributes']['size-total']
vol.space['space-guarantee-enabled'] =\
na_utils.to_bool(
v['volume-space-attributes'].get_child_content(

2
cinder/volume/drivers/netapp/utils.py
View File

@ -134,7 +134,7 @@ def resolve_hostname(hostname):
return sockaddr[0]
def round_down(value, precision):
def round_down(value, precision='0.00'):
return float(decimal.Decimal(six.text_type(value)).quantize(
decimal.Decimal(precision), rounding=decimal.ROUND_DOWN))