From f6adf72d5e162b52853cd0222db67cb485f929b3 Mon Sep 17 00:00:00 2001 From: "Erlon R. Cruz" Date: Tue, 12 Jul 2016 11:36:24 -0300 Subject: [PATCH] Fix HNAS stats reporting HNAS driver pools are not reporting thin_provisioning to scheduler, so when creating a volume, is not possible to limit the usage of the backend space. Change-Id: I16c3b4025b841a122d76ae4b7a44667c999236e2 Partial-bug: #1600295 --- .../hitachi/test_hitachi_hnas_backend.py | 53 +++++++++-------- .../hitachi/test_hitachi_hnas_iscsi.py | 3 +- .../drivers/hitachi/test_hitachi_hnas_nfs.py | 2 +- cinder/volume/drivers/hitachi/hnas_backend.py | 57 +++++++++++++------ cinder/volume/drivers/hitachi/hnas_iscsi.py | 25 ++++++-- cinder/volume/drivers/hitachi/hnas_nfs.py | 18 ++++-- ...hnas-stats-reporting-1335e582e46ff440.yaml | 4 ++ 7 files changed, 106 insertions(+), 56 deletions(-) create mode 100644 releasenotes/notes/fix-hnas-stats-reporting-1335e582e46ff440.yaml diff --git a/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_backend.py b/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_backend.py index ced2469d9e4..eef3715632e 100644 --- a/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_backend.py +++ b/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_backend.py @@ -73,6 +73,15 @@ Thin ThinSize ThinAvail FS Type\n\ No 32 KB,WFS-2,128 DSBs\n\ \n" +df_f_tb = "\n\ +ID Label EVS Size Used Snapshots Deduped Avail \ +Thin ThinSize ThinAvail FS Type\n\ +---- ---------- --- ------ ------------ --------- ------- ------------ \ +---- -------- --------- --------------------\n\ +1025 fs-cinder 2 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \ + No 32 KB,WFS-2,128 DSBs\n\ +\n" + nfs_export = "\n\ Export name: /export01-husvm \n\ Export path: /export01-husvm \n\ @@ -163,6 +172,9 @@ File System : fs-cinder \n\ File System Mounted : YES \n\ Logical Unit Mounted: No" +hnas_fs_list = "%(l1)s\n\n%(l2)s\n\n " % {'l1': iscsilu_list, + 'l2': iscsilu_list_tb} + add_targetsecret = "Target created successfully." iscsi_target_list = "\n\ @@ -454,7 +466,9 @@ class HDSHNASBackendTest(test.TestCase): def test_get_fs_info(self): self.mock_object(self.hnas_backend, '_run_cmd', - mock.Mock(return_value=(df_f, ''))) + mock.Mock(side_effect=[(df_f, ''), + (evsfs_list, ''), + (hnas_fs_list, '')])) out = self.hnas_backend.get_fs_info('fs-cinder') @@ -462,7 +476,7 @@ class HDSHNASBackendTest(test.TestCase): self.assertEqual('fs-cinder', out['label']) self.assertEqual('228', out['available_size']) self.assertEqual('250', out['total_size']) - self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') + self.assertEqual(2050.0, out['provisioned_capacity']) def test_get_fs_empty_return(self): self.mock_object(self.hnas_backend, '_run_cmd', @@ -473,59 +487,48 @@ class HDSHNASBackendTest(test.TestCase): def test_get_fs_info_single_evs(self): self.mock_object(self.hnas_backend, '_run_cmd', - mock.Mock(return_value=(df_f_single_evs, ''))) + mock.Mock(side_effect=[(df_f_single_evs, ''), + (evsfs_list, ''), + (hnas_fs_list, '')])) out = self.hnas_backend.get_fs_info('fs-cinder') self.assertEqual('fs-cinder', out['label']) self.assertEqual('228', out['available_size']) self.assertEqual('250', out['total_size']) - self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') + self.assertEqual(2050.0, out['provisioned_capacity']) def test_get_fs_tb(self): available_size = float(228 * 1024 ** 2) total_size = float(250 * 1024 ** 2) - df_f_tb = "\n\ -ID Label EVS Size Used Snapshots Deduped Avail \ -Thin ThinSize ThinAvail FS Type\n\ ----- ---------- --- ------ ------------ --------- ------- ------------ \ ----- -------- --------- --------------------\n\ -1025 fs-cinder 2 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \ - No 32 KB,WFS-2,128 DSBs\n\ -\n" self.mock_object(self.hnas_backend, '_run_cmd', - mock.Mock(return_value=(df_f_tb, ''))) + mock.Mock(side_effect=[(df_f_tb, ''), + (evsfs_list, ''), + (hnas_fs_list, '')])) out = self.hnas_backend.get_fs_info('fs-cinder') - self.assertEqual('2', out['evs_id']) self.assertEqual('fs-cinder', out['label']) self.assertEqual(str(available_size), out['available_size']) self.assertEqual(str(total_size), out['total_size']) - self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') + self.assertEqual(2050.0, out['provisioned_capacity']) def test_get_fs_single_evs_tb(self): available_size = float(228 * 1024 ** 2) total_size = float(250 * 1024 ** 2) - df_f_tb = "\n\ -ID Label Size Used Snapshots Deduped Avail \ -Thin ThinSize ThinAvail FS Type\n\ ----- ---------- ------ ------------ --------- ------- ------------ \ ----- -------- --------- --------------------\n\ -1025 fs-cinder 250 TB 21.4 TB (9%) 0 B (0%) NA 228 TB (91%) \ - No 32 KB,WFS-2,128 DSBs\n\ -\n" self.mock_object(self.hnas_backend, '_run_cmd', - mock.Mock(return_value=(df_f_tb, ''))) + mock.Mock(side_effect=[(df_f_tb, ''), + (evsfs_list, ''), + (hnas_fs_list, '')])) out = self.hnas_backend.get_fs_info('fs-cinder') self.assertEqual('fs-cinder', out['label']) self.assertEqual(str(available_size), out['available_size']) self.assertEqual(str(total_size), out['total_size']) - self.hnas_backend._run_cmd.assert_called_with('df', '-af', 'fs-cinder') + self.assertEqual(2050.0, out['provisioned_capacity']) def test_create_lu(self): self.mock_object(self.hnas_backend, '_run_cmd', diff --git a/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_iscsi.py b/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_iscsi.py index f8b4f980e89..c6ecbf3f487 100644 --- a/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_iscsi.py +++ b/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_iscsi.py @@ -384,7 +384,8 @@ class HNASiSCSIDriverTest(test.TestCase): 'label': 'fs-cinder', 'available_size': '228', 'used_size': '21.4', - 'id': '1025' + 'id': '1025', + 'provisioned_capacity': 0.0 } self.mock_object(HNASSSHBackend, 'get_fs_info', diff --git a/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_nfs.py b/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_nfs.py index 8efc6190dfa..9fee4e134d0 100644 --- a/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_nfs.py +++ b/cinder/tests/unit/volume/drivers/hitachi/test_hitachi_hnas_nfs.py @@ -290,7 +290,7 @@ class HNASNFSDriverTest(test.TestCase): self.mock_object(self.driver, '_update_volume_stats') self.mock_object(self.driver, '_get_capacity_info', - mock.Mock(return_value=(150, 50, 100))) + return_value=(150, 50, 100)) out = self.driver.get_volume_stats() diff --git a/cinder/volume/drivers/hitachi/hnas_backend.py b/cinder/volume/drivers/hitachi/hnas_backend.py index 76ddc93451a..ad765161ede 100644 --- a/cinder/volume/drivers/hitachi/hnas_backend.py +++ b/cinder/volume/drivers/hitachi/hnas_backend.py @@ -220,6 +220,20 @@ class HNASSSHBackend(object): fs_info['available_size'] = _convert_size( fs_info['available_size']) + # Get the iSCSI LUs in the FS + evs_id = self.get_evs(fs_label) + out, err = self._run_cmd('console-context', '--evs', evs_id, + 'iscsi-lu', 'list') + all_lus = [self._parse_lu_info(lu_raw) + for lu_raw in out.split('\n\n')[:-1]] + + provisioned_cap = 0 + for lu in all_lus: + if lu['filesystem'] == fs_label: + provisioned_cap += lu['size'] + + fs_info['provisioned_capacity'] = provisioned_cap + LOG.debug("File system info of %(fs)s (sizes in GB): %(info)s.", {'fs': fs_label, 'info': fs_info}) @@ -613,6 +627,29 @@ class HNASSSHBackend(object): return lu_info + def _parse_lu_info(self, output): + lu_info = {} + if 'does not exist.' not in output: + aux = output.split('\n') + lu_info['name'] = aux[0].split(':')[1].strip() + lu_info['comment'] = aux[1].split(':')[1].strip() + lu_info['path'] = aux[2].split(':')[1].strip() + lu_info['size'] = aux[3].split(':')[1].strip() + lu_info['filesystem'] = aux[4].split(':')[1].strip() + lu_info['fs_mounted'] = aux[5].split(':')[1].strip() + lu_info['lu_mounted'] = aux[6].split(':')[1].strip() + + if 'TB' in lu_info['size']: + sz_convert = float(lu_info['size'].split()[0]) * units.Ki + lu_info['size'] = sz_convert + elif 'MB' in lu_info['size']: + sz_convert = float(lu_info['size'].split()[0]) / units.Ki + lu_info['size'] = sz_convert + else: + lu_info['size'] = float(lu_info['size'].split()[0]) + + return lu_info + def get_existing_lu_info(self, lu_name, fs_label=None, evs_id=None): """Gets the information for the specified Logical Unit. @@ -635,30 +672,14 @@ class HNASSSHBackend(object): (mounted or not) } """ - lu_info = {} + if evs_id is None: evs_id = self.get_evs(fs_label) lu_name = "'{}'".format(lu_name) out, err = self._run_cmd("console-context", "--evs", evs_id, 'iscsi-lu', 'list', lu_name) - - if 'does not exist.' not in out: - aux = out.split('\n') - lu_info['name'] = aux[0].split(':')[1].strip() - lu_info['comment'] = aux[1].split(':')[1].strip() - lu_info['path'] = aux[2].split(':')[1].strip() - lu_info['size'] = aux[3].split(':')[1].strip() - lu_info['filesystem'] = aux[4].split(':')[1].strip() - lu_info['fs_mounted'] = aux[5].split(':')[1].strip() - lu_info['lu_mounted'] = aux[6].split(':')[1].strip() - - if 'TB' in lu_info['size']: - sz_convert = float(lu_info['size'].split()[0]) * units.Ki - lu_info['size'] = sz_convert - else: - lu_info['size'] = float(lu_info['size'].split()[0]) - + lu_info = self._parse_lu_info(out) LOG.debug('get_existing_lu_info: LU info: %(lu)s', {'lu': lu_info}) return lu_info diff --git a/cinder/volume/drivers/hitachi/hnas_iscsi.py b/cinder/volume/drivers/hitachi/hnas_iscsi.py index b95857505f1..a786f75f552 100644 --- a/cinder/volume/drivers/hitachi/hnas_iscsi.py +++ b/cinder/volume/drivers/hitachi/hnas_iscsi.py @@ -93,6 +93,7 @@ class HNASISCSIDriver(driver.ISCSIDriver): Updated to use versioned objects Changed the class name to HNASISCSIDriver Deprecated XML config file + Fixed driver stats reporting """ # ThirdPartySystems wiki page @@ -101,6 +102,7 @@ class HNASISCSIDriver(driver.ISCSIDriver): def __init__(self, *args, **kwargs): """Initializes and reads different config parameters.""" + super(HNASISCSIDriver, self).__init__(*args, **kwargs) self.configuration = kwargs.get('configuration', None) self.context = {} self.config = {} @@ -126,7 +128,11 @@ class HNASISCSIDriver(driver.ISCSIDriver): service_parameters, optional_parameters) - super(HNASISCSIDriver, self).__init__(*args, **kwargs) + self.reserved_percentage = ( + self.configuration.safe_get('reserved_percentage')) + self.max_osr = ( + self.configuration.safe_get('max_over_subscription_ratio')) + self.backend = hnas_backend.HNASSSHBackend(self.config) def _get_service(self, volume): @@ -274,6 +280,15 @@ class HNASISCSIDriver(driver.ISCSIDriver): """Get FS stats from HNAS. :returns: dictionary with the stats from HNAS + _stats['pools'] = { + 'total_capacity_gb': total size of the pool, + 'free_capacity_gb': the available size, + 'QoS_support': bool to indicate if QoS is supported, + 'reserved_percentage': percentage of size reserved, + 'max_over_subscription_ratio': oversubscription rate, + 'thin_provisioning_support': thin support (True), + 'reserved_percentage': reserved percentage + } """ hnas_stat = {} be_name = self.configuration.safe_get('volume_backend_name') @@ -281,17 +296,17 @@ class HNASISCSIDriver(driver.ISCSIDriver): hnas_stat["vendor_name"] = 'Hitachi' hnas_stat["driver_version"] = HNAS_ISCSI_VERSION hnas_stat["storage_protocol"] = 'iSCSI' - hnas_stat['reserved_percentage'] = 0 for pool in self.pools: fs_info = self.backend.get_fs_info(pool['fs']) - + pool['provisioned_capacity_gb'] = fs_info['provisioned_capacity'] pool['total_capacity_gb'] = (float(fs_info['total_size'])) pool['free_capacity_gb'] = ( float(fs_info['total_size']) - float(fs_info['used_size'])) - pool['allocated_capacity_gb'] = (float(fs_info['total_size'])) pool['QoS_support'] = 'False' - pool['reserved_percentage'] = 0 + pool['reserved_percentage'] = self.reserved_percentage + pool['max_over_subscription_ratio'] = self.max_osr + pool['thin_provisioning_support'] = True hnas_stat['pools'] = self.pools diff --git a/cinder/volume/drivers/hitachi/hnas_nfs.py b/cinder/volume/drivers/hitachi/hnas_nfs.py index c393493f4d7..8edb0aae6fa 100644 --- a/cinder/volume/drivers/hitachi/hnas_nfs.py +++ b/cinder/volume/drivers/hitachi/hnas_nfs.py @@ -80,6 +80,7 @@ class HNASNFSDriver(nfs.NfsDriver): Changed the class name to HNASNFSDriver Deprecated XML config file Added support to manage/unmanage snapshots features + Fixed driver stats reporting """ # ThirdPartySystems wiki page CI_WIKI_NAME = "Hitachi_HNAS_CI" @@ -274,12 +275,13 @@ class HNASNFSDriver(nfs.NfsDriver): :param refresh: if it is True, update the stats first. :returns: dictionary with the stats from HNAS - _stats['pools']={ + _stats['pools'] = { 'total_capacity_gb': total size of the pool, 'free_capacity_gb': the available size, - 'allocated_capacity_gb': current allocated size, 'QoS_support': bool to indicate if QoS is supported, - 'reserved_percentage': percentage of size reserved + 'reserved_percentage': percentage of size reserved, + 'max_over_subscription_ratio': oversubscription rate, + 'thin_provisioning_support': thin support (True), } """ LOG.info(_LI("Getting volume stats")) @@ -289,13 +291,17 @@ class HNASNFSDriver(nfs.NfsDriver): _stats["driver_version"] = HNAS_NFS_VERSION _stats["storage_protocol"] = 'NFS' + max_osr = self.max_over_subscription_ratio + for pool in self.pools: - capacity, free, used = self._get_capacity_info(pool['fs']) + capacity, free, provisioned = self._get_capacity_info(pool['fs']) pool['total_capacity_gb'] = capacity / float(units.Gi) pool['free_capacity_gb'] = free / float(units.Gi) - pool['allocated_capacity_gb'] = used / float(units.Gi) + pool['provisioned_capacity_gb'] = provisioned / float(units.Gi) pool['QoS_support'] = 'False' - pool['reserved_percentage'] = 0 + pool['reserved_percentage'] = self.reserved_percentage + pool['max_over_subscription_ratio'] = max_osr + pool['thin_provisioning_support'] = True _stats['pools'] = self.pools diff --git a/releasenotes/notes/fix-hnas-stats-reporting-1335e582e46ff440.yaml b/releasenotes/notes/fix-hnas-stats-reporting-1335e582e46ff440.yaml new file mode 100644 index 00000000000..897a3a1a8be --- /dev/null +++ b/releasenotes/notes/fix-hnas-stats-reporting-1335e582e46ff440.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - Fixes HNAS stats reporting. HNAS driver was not correctly reporting THIN + provisioning and related stats.