openstack/deb-cinder
Code Issues Proposed changes

Fix HNAS stats reporting

Browse Source 
HNAS driver pools are not reporting thin_provisioning to scheduler,
so when creating a volume, is not possible to limit the usage of
the backend space.

Change-Id: I16c3b4025b841a122d76ae4b7a44667c999236e2
Partial-bug: #1600295
This commit is contained in:
Erlon R. Cruz 2016-07-12 11:36:24 -03:00
parent 1244318fe2
commit f6adf72d5e
7 changed files with 106 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_backend.py
View File

@ -73,6 +73,15 @@ Thin ThinSize ThinAvail FS Type\n\
No 32 KB,WFS-2,128 DSBs\n\ No 32 KB,WFS-2,128 DSBs\n\
\n" \n"
df_f_tb = "\n\
ID Label EVS Size Used Snapshots Deduped Avail \
Thin ThinSize ThinAvail FS Type\n\
---- ---------- --- ------ ------------ --------- ------- ------------ \
---- -------- --------- --------------------\n\
1025 fs-cinder 2 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \
No 32 KB,WFS-2,128 DSBs\n\
\n"
nfs_export = "\n\ nfs_export = "\n\
Export name: /export01-husvm \n\ Export name: /export01-husvm \n\
Export path: /export01-husvm \n\ Export path: /export01-husvm \n\
@ -163,6 +172,9 @@ File System : fs-cinder \n\
File System Mounted : YES \n\ File System Mounted : YES \n\
Logical Unit Mounted: No" Logical Unit Mounted: No"
hnas_fs_list = "%(l1)s\n\n%(l2)s\n\n " % {'l1': iscsilu_list,
'l2': iscsilu_list_tb}
add_targetsecret = "Target created successfully." add_targetsecret = "Target created successfully."
iscsi_target_list = "\n\ iscsi_target_list = "\n\
@ -454,7 +466,9 @@ class HDSHNASBackendTest(test.TestCase):
def test_get_fs_info(self): def test_get_fs_info(self):
self.mock_object(self.hnas_backend, '_run_cmd', self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f, ''))) mock.Mock(side_effect=[(df_f, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder') out = self.hnas_backend.get_fs_info('fs-cinder')
@ -462,7 +476,7 @@ class HDSHNASBackendTest(test.TestCase):
self.assertEqual('fs-cinder', out['label']) self.assertEqual('fs-cinder', out['label'])
self.assertEqual('228', out['available_size']) self.assertEqual('228', out['available_size'])
self.assertEqual('250', out['total_size']) self.assertEqual('250', out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') self.assertEqual(2050.0, out['provisioned_capacity'])
def test_get_fs_empty_return(self): def test_get_fs_empty_return(self):
self.mock_object(self.hnas_backend, '_run_cmd', self.mock_object(self.hnas_backend, '_run_cmd',
@ -473,59 +487,48 @@ class HDSHNASBackendTest(test.TestCase):
def test_get_fs_info_single_evs(self): def test_get_fs_info_single_evs(self):
self.mock_object(self.hnas_backend, '_run_cmd', self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f_single_evs, ''))) mock.Mock(side_effect=[(df_f_single_evs, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder') out = self.hnas_backend.get_fs_info('fs-cinder')
self.assertEqual('fs-cinder', out['label']) self.assertEqual('fs-cinder', out['label'])
self.assertEqual('228', out['available_size']) self.assertEqual('228', out['available_size'])
self.assertEqual('250', out['total_size']) self.assertEqual('250', out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') self.assertEqual(2050.0, out['provisioned_capacity'])
def test_get_fs_tb(self): def test_get_fs_tb(self):
available_size = float(228 * 1024 ** 2) available_size = float(228 * 1024 ** 2)
total_size = float(250 * 1024 ** 2) total_size = float(250 * 1024 ** 2)
df_f_tb = "\n\
ID Label EVS Size Used Snapshots Deduped Avail \
Thin ThinSize ThinAvail FS Type\n\
---- ---------- --- ------ ------------ --------- ------- ------------ \
---- -------- --------- --------------------\n\
1025 fs-cinder 2 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \
No 32 KB,WFS-2,128 DSBs\n\
\n"
self.mock_object(self.hnas_backend, '_run_cmd', self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f_tb, ''))) mock.Mock(side_effect=[(df_f_tb, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder') out = self.hnas_backend.get_fs_info('fs-cinder')
self.assertEqual('2', out['evs_id'])
self.assertEqual('fs-cinder', out['label']) self.assertEqual('fs-cinder', out['label'])
self.assertEqual(str(available_size), out['available_size']) self.assertEqual(str(available_size), out['available_size'])
self.assertEqual(str(total_size), out['total_size']) self.assertEqual(str(total_size), out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') self.assertEqual(2050.0, out['provisioned_capacity'])
def test_get_fs_single_evs_tb(self): def test_get_fs_single_evs_tb(self):
available_size = float(228 * 1024 ** 2) available_size = float(228 * 1024 ** 2)
total_size = float(250 * 1024 ** 2) total_size = float(250 * 1024 ** 2)
df_f_tb = "\n\
ID Label Size Used Snapshots Deduped Avail \
Thin ThinSize ThinAvail FS Type\n\
---- ---------- ------ ------------ --------- ------- ------------ \
---- -------- --------- --------------------\n\
1025 fs-cinder 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \
No 32 KB,WFS-2,128 DSBs\n\
\n"
self.mock_object(self.hnas_backend, '_run_cmd', self.mock_object(self.hnas_backend, '_run_cmd',
mock.Mock(return_value=(df_f_tb, ''))) mock.Mock(side_effect=[(df_f_tb, ''),
(evsfs_list, ''),
(hnas_fs_list, '')]))
out = self.hnas_backend.get_fs_info('fs-cinder') out = self.hnas_backend.get_fs_info('fs-cinder')
self.assertEqual('fs-cinder', out['label']) self.assertEqual('fs-cinder', out['label'])
self.assertEqual(str(available_size), out['available_size']) self.assertEqual(str(available_size), out['available_size'])
self.assertEqual(str(total_size), out['total_size']) self.assertEqual(str(total_size), out['total_size'])
self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') self.assertEqual(2050.0, out['provisioned_capacity'])
def test_create_lu(self): def test_create_lu(self):
self.mock_object(self.hnas_backend, '_run_cmd', self.mock_object(self.hnas_backend, '_run_cmd',

3
cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_iscsi.py
View File

@ -384,7 +384,8 @@ class HNASiSCSIDriverTest(test.TestCase):
'label': 'fs-cinder', 'label': 'fs-cinder',
'available_size': '228', 'available_size': '228',
'used_size': '21.4', 'used_size': '21.4',
'id': '1025' 'id': '1025',
'provisioned_capacity': 0.0
} }
self.mock_object(HNASSSHBackend, 'get_fs_info', self.mock_object(HNASSSHBackend, 'get_fs_info',

2
cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_nfs.py
View File

@ -290,7 +290,7 @@ class HNASNFSDriverTest(test.TestCase):
self.mock_object(self.driver, '_update_volume_stats') self.mock_object(self.driver, '_update_volume_stats')
self.mock_object(self.driver, '_get_capacity_info', self.mock_object(self.driver, '_get_capacity_info',
mock.Mock(return_value=(150, 50, 100))) return_value=(150, 50, 100))
out = self.driver.get_volume_stats() out = self.driver.get_volume_stats()

57
cinder/volume/drivers/hitachi/hnas_backend.py
View File

@ -220,6 +220,20 @@ class HNASSSHBackend(object):
fs_info['available_size'] = _convert_size( fs_info['available_size'] = _convert_size(
fs_info['available_size']) fs_info['available_size'])
# Get the iSCSI LUs in the FS
evs_id = self.get_evs(fs_label)
out, err = self._run_cmd('console-context', '--evs', evs_id,
'iscsi-lu', 'list')
all_lus = [self._parse_lu_info(lu_raw)
for lu_raw in out.split('\n\n')[:-1]]
provisioned_cap = 0
for lu in all_lus:
if lu['filesystem'] == fs_label:
provisioned_cap += lu['size']
fs_info['provisioned_capacity'] = provisioned_cap
LOG.debug("File system info of %(fs)s (sizes in GB): %(info)s.", LOG.debug("File system info of %(fs)s (sizes in GB): %(info)s.",
{'fs': fs_label, 'info': fs_info}) {'fs': fs_label, 'info': fs_info})
@ -613,6 +627,29 @@ class HNASSSHBackend(object):
return lu_info return lu_info
def _parse_lu_info(self, output):
lu_info = {}
if 'does not exist.' not in output:
aux = output.split('\n')
lu_info['name'] = aux[0].split(':')[1].strip()
lu_info['comment'] = aux[1].split(':')[1].strip()
lu_info['path'] = aux[2].split(':')[1].strip()
lu_info['size'] = aux[3].split(':')[1].strip()
lu_info['filesystem'] = aux[4].split(':')[1].strip()
lu_info['fs_mounted'] = aux[5].split(':')[1].strip()
lu_info['lu_mounted'] = aux[6].split(':')[1].strip()
if 'TB' in lu_info['size']:
sz_convert = float(lu_info['size'].split()[0]) * units.Ki
lu_info['size'] = sz_convert
elif 'MB' in lu_info['size']:
sz_convert = float(lu_info['size'].split()[0]) / units.Ki
lu_info['size'] = sz_convert
else:
lu_info['size'] = float(lu_info['size'].split()[0])
return lu_info
def get_existing_lu_info(self, lu_name, fs_label=None, evs_id=None): def get_existing_lu_info(self, lu_name, fs_label=None, evs_id=None):
"""Gets the information for the specified Logical Unit. """Gets the information for the specified Logical Unit.
@ -635,30 +672,14 @@ class HNASSSHBackend(object):
(mounted or not) (mounted or not)
} }
""" """
lu_info = {}
if evs_id is None: if evs_id is None:
evs_id = self.get_evs(fs_label) evs_id = self.get_evs(fs_label)
lu_name = "'{}'".format(lu_name) lu_name = "'{}'".format(lu_name)
out, err = self._run_cmd("console-context", "--evs", evs_id, out, err = self._run_cmd("console-context", "--evs", evs_id,
'iscsi-lu', 'list', lu_name) 'iscsi-lu', 'list', lu_name)
lu_info = self._parse_lu_info(out)
if 'does not exist.' not in out:
aux = out.split('\n')
lu_info['name'] = aux[0].split(':')[1].strip()
lu_info['comment'] = aux[1].split(':')[1].strip()
lu_info['path'] = aux[2].split(':')[1].strip()
lu_info['size'] = aux[3].split(':')[1].strip()
lu_info['filesystem'] = aux[4].split(':')[1].strip()
lu_info['fs_mounted'] = aux[5].split(':')[1].strip()
lu_info['lu_mounted'] = aux[6].split(':')[1].strip()
if 'TB' in lu_info['size']:
sz_convert = float(lu_info['size'].split()[0]) * units.Ki
lu_info['size'] = sz_convert
else:
lu_info['size'] = float(lu_info['size'].split()[0])
LOG.debug('get_existing_lu_info: LU info: %(lu)s', {'lu': lu_info}) LOG.debug('get_existing_lu_info: LU info: %(lu)s', {'lu': lu_info})
return lu_info return lu_info

25
cinder/volume/drivers/hitachi/hnas_iscsi.py
View File

@ -93,6 +93,7 @@ class HNASISCSIDriver(driver.ISCSIDriver):
Updated to use versioned objects Updated to use versioned objects
Changed the class name to HNASISCSIDriver Changed the class name to HNASISCSIDriver
Deprecated XML config file Deprecated XML config file
Fixed driver stats reporting
""" """
# ThirdPartySystems wiki page # ThirdPartySystems wiki page
@ -101,6 +102,7 @@ class HNASISCSIDriver(driver.ISCSIDriver):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
"""Initializes and reads different config parameters.""" """Initializes and reads different config parameters."""
super(HNASISCSIDriver, self).__init__(*args, **kwargs)
self.configuration = kwargs.get('configuration', None) self.configuration = kwargs.get('configuration', None)
self.context = {} self.context = {}
self.config = {} self.config = {}
@ -126,7 +128,11 @@ class HNASISCSIDriver(driver.ISCSIDriver):
service_parameters, service_parameters,
optional_parameters) optional_parameters)
super(HNASISCSIDriver, self).__init__(*args, **kwargs) self.reserved_percentage = (
self.configuration.safe_get('reserved_percentage'))
self.max_osr = (
self.configuration.safe_get('max_over_subscription_ratio'))
self.backend = hnas_backend.HNASSSHBackend(self.config) self.backend = hnas_backend.HNASSSHBackend(self.config)
def _get_service(self, volume): def _get_service(self, volume):
@ -274,6 +280,15 @@ class HNASISCSIDriver(driver.ISCSIDriver):
"""Get FS stats from HNAS. """Get FS stats from HNAS.
:returns: dictionary with the stats from HNAS :returns: dictionary with the stats from HNAS
_stats['pools'] = {
'total_capacity_gb': total size of the pool,
'free_capacity_gb': the available size,
'QoS_support': bool to indicate if QoS is supported,
'reserved_percentage': percentage of size reserved,
'max_over_subscription_ratio': oversubscription rate,
'thin_provisioning_support': thin support (True),
'reserved_percentage': reserved percentage
}
""" """
hnas_stat = {} hnas_stat = {}
be_name = self.configuration.safe_get('volume_backend_name') be_name = self.configuration.safe_get('volume_backend_name')
@ -281,17 +296,17 @@ class HNASISCSIDriver(driver.ISCSIDriver):
hnas_stat["vendor_name"] = 'Hitachi' hnas_stat["vendor_name"] = 'Hitachi'
hnas_stat["driver_version"] = HNAS_ISCSI_VERSION hnas_stat["driver_version"] = HNAS_ISCSI_VERSION
hnas_stat["storage_protocol"] = 'iSCSI' hnas_stat["storage_protocol"] = 'iSCSI'
hnas_stat['reserved_percentage'] = 0
for pool in self.pools: for pool in self.pools:
fs_info = self.backend.get_fs_info(pool['fs']) fs_info = self.backend.get_fs_info(pool['fs'])
pool['provisioned_capacity_gb'] = fs_info['provisioned_capacity']
pool['total_capacity_gb'] = (float(fs_info['total_size'])) pool['total_capacity_gb'] = (float(fs_info['total_size']))
pool['free_capacity_gb'] = ( pool['free_capacity_gb'] = (
float(fs_info['total_size']) - float(fs_info['used_size'])) float(fs_info['total_size']) - float(fs_info['used_size']))
pool['allocated_capacity_gb'] = (float(fs_info['total_size']))
pool['QoS_support'] = 'False' pool['QoS_support'] = 'False'
pool['reserved_percentage'] = 0 pool['reserved_percentage'] = self.reserved_percentage
pool['max_over_subscription_ratio'] = self.max_osr
pool['thin_provisioning_support'] = True
hnas_stat['pools'] = self.pools hnas_stat['pools'] = self.pools

16
cinder/volume/drivers/hitachi/hnas_nfs.py
View File

@ -80,6 +80,7 @@ class HNASNFSDriver(nfs.NfsDriver):
Changed the class name to HNASNFSDriver Changed the class name to HNASNFSDriver
Deprecated XML config file Deprecated XML config file
Added support to manage/unmanage snapshots features Added support to manage/unmanage snapshots features
Fixed driver stats reporting
""" """
# ThirdPartySystems wiki page # ThirdPartySystems wiki page
CI_WIKI_NAME = "Hitachi_HNAS_CI" CI_WIKI_NAME = "Hitachi_HNAS_CI"
@ -277,9 +278,10 @@ class HNASNFSDriver(nfs.NfsDriver):
_stats['pools'] = { _stats['pools'] = {
'total_capacity_gb': total size of the pool, 'total_capacity_gb': total size of the pool,
'free_capacity_gb': the available size, 'free_capacity_gb': the available size,
'allocated_capacity_gb': current allocated size,
'QoS_support': bool to indicate if QoS is supported, 'QoS_support': bool to indicate if QoS is supported,
'reserved_percentage': percentage of size reserved 'reserved_percentage': percentage of size reserved,
'max_over_subscription_ratio': oversubscription rate,
'thin_provisioning_support': thin support (True),
} }
""" """
LOG.info(_LI("Getting volume stats")) LOG.info(_LI("Getting volume stats"))
@ -289,13 +291,17 @@ class HNASNFSDriver(nfs.NfsDriver):
_stats["driver_version"] = HNAS_NFS_VERSION _stats["driver_version"] = HNAS_NFS_VERSION
_stats["storage_protocol"] = 'NFS' _stats["storage_protocol"] = 'NFS'
max_osr = self.max_over_subscription_ratio
for pool in self.pools: for pool in self.pools:
capacity, free, used = self._get_capacity_info(pool['fs']) capacity, free, provisioned = self._get_capacity_info(pool['fs'])
pool['total_capacity_gb'] = capacity / float(units.Gi) pool['total_capacity_gb'] = capacity / float(units.Gi)
pool['free_capacity_gb'] = free / float(units.Gi) pool['free_capacity_gb'] = free / float(units.Gi)
pool['allocated_capacity_gb'] = used / float(units.Gi) pool['provisioned_capacity_gb'] = provisioned / float(units.Gi)
pool['QoS_support'] = 'False' pool['QoS_support'] = 'False'
pool['reserved_percentage'] = 0 pool['reserved_percentage'] = self.reserved_percentage
pool['max_over_subscription_ratio'] = max_osr
pool['thin_provisioning_support'] = True
_stats['pools'] = self.pools _stats['pools'] = self.pools

4
releasenotes/notes/fix-hnas-stats-reporting-1335e582e46ff440.yaml Normal file
View File

@ -0,0 +1,4 @@
---
fixes:
- Fixes HNAS stats reporting. HNAS driver was not correctly reporting THIN
provisioning and related stats.