openstack/deb-cinder
Code Issues Proposed changes

Fix HNAS stats reporting

Browse Source 
HNAS driver pools are not reporting thin_provisioning to scheduler,
so when creating a volume, is not possible to limit the usage of
the backend space.

Change-Id: I16c3b4025b841a122d76ae4b7a44667c999236e2
Partial-bug: #1600295
This commit is contained in:
Erlon R. Cruz 2016-07-12 11:36:24 -03:00
parent 1244318fe2
commit f6adf72d5e
7 changed files with 106 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_backend.py
View File

@ -73,6 +73,15 @@ Thin ThinSize ThinAvail FS Type\n\
No 32 KB,WFS-2,128 DSBs\n\
\n"
df_f_tb = "\n\
ID Label EVS Size Used Snapshots Deduped Avail \
Thin ThinSize ThinAvail FS Type\n\
---- ---------- --- ------ ------------ --------- ------- ------------ \
---- -------- --------- --------------------\n\
1025 fs-cinder 2 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \
No 32 KB,WFS-2,128 DSBs\n\
\n"
nfs_export = "\n\
Export name: /export01-husvm \n\
Export path: /export01-husvm \n\
@ -163,6 +172,9 @@ File System : fs-cinder \n\
File System Mounted : YES \n\
Logical Unit Mounted: No"
hnas_fs_list = "%(l1)s\n\n%(l2)s\n\n " % {'l1': iscsilu_list,
'l2': iscsilu_list_tb}
add_targetsecret = "Target created successfully."
iscsi_target_list = "\n\
@ -454,7 +466,9 @@ class HDSHNASBackendTest(test.TestCase):
def test_get_fs_info(self):
self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f, '')))
mock.Mock(side_effect=[(df_f, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder')
@ -462,7 +476,7 @@ class HDSHNASBackendTest(test.TestCase):
self.assertEqual('fs-cinder', out['label'])
self.assertEqual('228', out['available_size'])
self.assertEqual('250', out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder')
self.assertEqual(2050.0, out['provisioned_capacity'])
def test_get_fs_empty_return(self):
self.mock_object(self.hnas_backend, '_run_cmd',
@ -473,59 +487,48 @@ class HDSHNASBackendTest(test.TestCase):
def test_get_fs_info_single_evs(self):
self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f_single_evs, '')))
mock.Mock(side_effect=[(df_f_single_evs, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder')
self.assertEqual('fs-cinder', out['label'])
self.assertEqual('228', out['available_size'])
self.assertEqual('250', out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder')
self.assertEqual(2050.0, out['provisioned_capacity'])
def test_get_fs_tb(self):
available_size = float(228 * 1024 ** 2)
total_size = float(250 * 1024 ** 2)
df_f_tb = "\n\
ID Label EVS Size Used Snapshots Deduped Avail \
Thin ThinSize ThinAvail FS Type\n\
---- ---------- --- ------ ------------ --------- ------- ------------ \
---- -------- --------- --------------------\n\
1025 fs-cinder 2 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \
No 32 KB,WFS-2,128 DSBs\n\
\n"
self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f_tb, '')))
mock.Mock(side_effect=[(df_f_tb, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder')
self.assertEqual('2', out['evs_id'])
self.assertEqual('fs-cinder', out['label'])
self.assertEqual(str(available_size), out['available_size'])
self.assertEqual(str(total_size), out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder')
self.assertEqual(2050.0, out['provisioned_capacity'])
def test_get_fs_single_evs_tb(self):
available_size = float(228 * 1024 ** 2)
total_size = float(250 * 1024 ** 2)
df_f_tb = "\n\
ID Label Size Used Snapshots Deduped Avail \
Thin ThinSize ThinAvail FS Type\n\
---- ---------- ------ ------------ --------- ------- ------------ \
---- -------- --------- --------------------\n\
1025 fs-cinder 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \
No 32 KB,WFS-2,128 DSBs\n\
\n"
self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f_tb, '')))
mock.Mock(side_effect=[(df_f_tb, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder')
self.assertEqual('fs-cinder', out['label'])
self.assertEqual(str(available_size), out['available_size'])
self.assertEqual(str(total_size), out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder')
self.assertEqual(2050.0, out['provisioned_capacity'])
def test_create_lu(self):
self.mock_object(self.hnas_backend, '_run_cmd',

3
cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_iscsi.py
View File

@ -384,7 +384,8 @@ class HNASiSCSIDriverTest(test.TestCase):
'label': 'fs-cinder',
'available_size': '228',
'used_size': '21.4',
'id': '1025'
'id': '1025',
'provisioned_capacity': 0.0
}
self.mock_object(HNASSSHBackend, 'get_fs_info',

2
cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_nfs.py
View File

@ -290,7 +290,7 @@ class HNASNFSDriverTest(test.TestCase):
self.mock_object(self.driver, '_update_volume_stats')
self.mock_object(self.driver, '_get_capacity_info',
mock.Mock(return_value=(150, 50, 100)))
return_value=(150, 50, 100))
out = self.driver.get_volume_stats()

57
cinder/volume/drivers/hitachi/hnas_backend.py
View File

@ -220,6 +220,20 @@ class HNASSSHBackend(object):
fs_info['available_size'] = _convert_size(
fs_info['available_size'])
# Get the iSCSI LUs in the FS
evs_id = self.get_evs(fs_label)
out, err = self._run_cmd('console-context', '--evs', evs_id,
'iscsi-lu', 'list')
all_lus = [self._parse_lu_info(lu_raw)
for lu_raw in out.split('\n\n')[:-1]]
provisioned_cap = 0
for lu in all_lus:
if lu['filesystem'] == fs_label:
provisioned_cap += lu['size']
fs_info['provisioned_capacity'] = provisioned_cap
LOG.debug("File system info of %(fs)s (sizes in GB): %(info)s.",
{'fs': fs_label, 'info': fs_info})
@ -613,6 +627,29 @@ class HNASSSHBackend(object):
return lu_info
def _parse_lu_info(self, output):
lu_info = {}
if 'does not exist.' not in output:
aux = output.split('\n')
lu_info['name'] = aux[0].split(':')[1].strip()
lu_info['comment'] = aux[1].split(':')[1].strip()
lu_info['path'] = aux[2].split(':')[1].strip()
lu_info['size'] = aux[3].split(':')[1].strip()
lu_info['filesystem'] = aux[4].split(':')[1].strip()
lu_info['fs_mounted'] = aux[5].split(':')[1].strip()
lu_info['lu_mounted'] = aux[6].split(':')[1].strip()
if 'TB' in lu_info['size']:
sz_convert = float(lu_info['size'].split()[0]) * units.Ki
lu_info['size'] = sz_convert
elif 'MB' in lu_info['size']:
sz_convert = float(lu_info['size'].split()[0]) / units.Ki
lu_info['size'] = sz_convert
else:
lu_info['size'] = float(lu_info['size'].split()[0])
return lu_info
def get_existing_lu_info(self, lu_name, fs_label=None, evs_id=None):
"""Gets the information for the specified Logical Unit.
@ -635,30 +672,14 @@ class HNASSSHBackend(object):
(mounted or not)
}
"""
lu_info = {}
if evs_id is None:
evs_id = self.get_evs(fs_label)
lu_name = "'{}'".format(lu_name)
out, err = self._run_cmd("console-context", "--evs", evs_id,
'iscsi-lu', 'list', lu_name)
if 'does not exist.' not in out:
aux = out.split('\n')
lu_info['name'] = aux[0].split(':')[1].strip()
lu_info['comment'] = aux[1].split(':')[1].strip()
lu_info['path'] = aux[2].split(':')[1].strip()
lu_info['size'] = aux[3].split(':')[1].strip()
lu_info['filesystem'] = aux[4].split(':')[1].strip()
lu_info['fs_mounted'] = aux[5].split(':')[1].strip()
lu_info['lu_mounted'] = aux[6].split(':')[1].strip()
if 'TB' in lu_info['size']:
sz_convert = float(lu_info['size'].split()[0]) * units.Ki
lu_info['size'] = sz_convert
else:
lu_info['size'] = float(lu_info['size'].split()[0])
lu_info = self._parse_lu_info(out)
LOG.debug('get_existing_lu_info: LU info: %(lu)s', {'lu': lu_info})
return lu_info

25
cinder/volume/drivers/hitachi/hnas_iscsi.py
View File

@ -93,6 +93,7 @@ class HNASISCSIDriver(driver.ISCSIDriver):
Updated to use versioned objects
Changed the class name to HNASISCSIDriver
Deprecated XML config file
Fixed driver stats reporting
"""
# ThirdPartySystems wiki page
@ -101,6 +102,7 @@ class HNASISCSIDriver(driver.ISCSIDriver):
def __init__(self, *args, **kwargs):
"""Initializes and reads different config parameters."""
super(HNASISCSIDriver, self).__init__(*args, **kwargs)
self.configuration = kwargs.get('configuration', None)
self.context = {}
self.config = {}
@ -126,7 +128,11 @@ class HNASISCSIDriver(driver.ISCSIDriver):
service_parameters,
optional_parameters)
super(HNASISCSIDriver, self).__init__(*args, **kwargs)
self.reserved_percentage = (
self.configuration.safe_get('reserved_percentage'))
self.max_osr = (
self.configuration.safe_get('max_over_subscription_ratio'))
self.backend = hnas_backend.HNASSSHBackend(self.config)
def _get_service(self, volume):
@ -274,6 +280,15 @@ class HNASISCSIDriver(driver.ISCSIDriver):
"""Get FS stats from HNAS.
:returns: dictionary with the stats from HNAS
_stats['pools'] = {
'total_capacity_gb': total size of the pool,
'free_capacity_gb': the available size,
'QoS_support': bool to indicate if QoS is supported,
'reserved_percentage': percentage of size reserved,
'max_over_subscription_ratio': oversubscription rate,
'thin_provisioning_support': thin support (True),
'reserved_percentage': reserved percentage
}
"""
hnas_stat = {}
be_name = self.configuration.safe_get('volume_backend_name')
@ -281,17 +296,17 @@ class HNASISCSIDriver(driver.ISCSIDriver):
hnas_stat["vendor_name"] = 'Hitachi'
hnas_stat["driver_version"] = HNAS_ISCSI_VERSION
hnas_stat["storage_protocol"] = 'iSCSI'
hnas_stat['reserved_percentage'] = 0
for pool in self.pools:
fs_info = self.backend.get_fs_info(pool['fs'])
pool['provisioned_capacity_gb'] = fs_info['provisioned_capacity']
pool['total_capacity_gb'] = (float(fs_info['total_size']))
pool['free_capacity_gb'] = (
float(fs_info['total_size']) - float(fs_info['used_size']))
pool['allocated_capacity_gb'] = (float(fs_info['total_size']))
pool['QoS_support'] = 'False'
pool['reserved_percentage'] = 0
pool['reserved_percentage'] = self.reserved_percentage
pool['max_over_subscription_ratio'] = self.max_osr
pool['thin_provisioning_support'] = True
hnas_stat['pools'] = self.pools

18
cinder/volume/drivers/hitachi/hnas_nfs.py
View File

@ -80,6 +80,7 @@ class HNASNFSDriver(nfs.NfsDriver):
Changed the class name to HNASNFSDriver
Deprecated XML config file
Added support to manage/unmanage snapshots features
Fixed driver stats reporting
"""
# ThirdPartySystems wiki page
CI_WIKI_NAME = "Hitachi_HNAS_CI"
@ -274,12 +275,13 @@ class HNASNFSDriver(nfs.NfsDriver):
:param refresh: if it is True, update the stats first.
:returns: dictionary with the stats from HNAS
_stats['pools']={
_stats['pools'] = {
'total_capacity_gb': total size of the pool,
'free_capacity_gb': the available size,
'allocated_capacity_gb': current allocated size,
'QoS_support': bool to indicate if QoS is supported,
'reserved_percentage': percentage of size reserved
'reserved_percentage': percentage of size reserved,
'max_over_subscription_ratio': oversubscription rate,
'thin_provisioning_support': thin support (True),
}
"""
LOG.info(_LI("Getting volume stats"))
@ -289,13 +291,17 @@ class HNASNFSDriver(nfs.NfsDriver):
_stats["driver_version"] = HNAS_NFS_VERSION
_stats["storage_protocol"] = 'NFS'
max_osr = self.max_over_subscription_ratio
for pool in self.pools:
capacity, free, used = self._get_capacity_info(pool['fs'])
capacity, free, provisioned = self._get_capacity_info(pool['fs'])
pool['total_capacity_gb'] = capacity / float(units.Gi)
pool['free_capacity_gb'] = free / float(units.Gi)
pool['allocated_capacity_gb'] = used / float(units.Gi)
pool['provisioned_capacity_gb'] = provisioned / float(units.Gi)
pool['QoS_support'] = 'False'
pool['reserved_percentage'] = 0
pool['reserved_percentage'] = self.reserved_percentage
pool['max_over_subscription_ratio'] = max_osr
pool['thin_provisioning_support'] = True
_stats['pools'] = self.pools

4
releasenotes/notes/fix-hnas-stats-reporting-1335e582e46ff440.yaml Normal file
View File

@ -0,0 +1,4 @@
---
fixes:
- Fixes HNAS stats reporting. HNAS driver was not correctly reporting THIN
provisioning and related stats.