[SVf] Add cleanrate in volume-type extra specs

[Spectrum Virtualize family] Added support to pass cleanrate
parameter in volume-type specs. Cleanrate parameter can now be
passed as an extra spec in volume-type or fetched from cinder.conf.
If these are not provided then it will take the default value.

Implements: blueprint ibm-svf-cleanrate-spec

Change-Id: I0a373cd130420ebe775e05db538868a4d333db1b

cinder/tests/unit/volume/drivers/ibm/test_storwize_svc.py

@@ -4588,6 +4588,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
                                SVC_POOLS,
                                'storwize_svc_flashcopy_timeout': 20,
                                'storwize_svc_flashcopy_rate': 49,
+                               'storwize_svc_clean_rate': 50,
                                'storwize_svc_allow_tenant_qos': True}
             config = conf.Configuration(storwize_svc_common.storwize_svc_opts,
                                         conf.SHARED_CONF_GROUP)
@@ -5128,6 +5129,7 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
                'stretched_cluster': None,
                'nofmtdisk': False,
                'flashcopy_rate': 49,
+               'clean_rate': 50,
                'mirror_pool': None,
                'volume_topology': None,
                'peer_pool': None,
@@ -5184,22 +5186,30 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         self._assert_vol_exists(snap1['name'], False)
         self._reset_flags()
 
-        # Test falshcopy_rate > 100 on 7.2.0.0
+        # Test flashcopy_rate > 100 on 7.2.0.0
         self._set_flag('storwize_svc_flashcopy_rate', 149)
         self.assertRaises(exception.VolumeDriverException,
                           self.driver.create_snapshot, snap1)
         self._assert_vol_exists(snap1['name'], False)
         self._reset_flags()
 
-        # Test falshcopy_rate out of range
+        # Test clean_rate < 150 on 7.2.0.0
+        self._set_flag('storwize_svc_clean_rate', 100)
+        vol2 = self._create_volume()
+        snap2 = self._generate_snap_info(vol2.id)
+        self.driver.create_snapshot(snap2)
+        self._assert_vol_exists(snap2['name'], True)
+        self._reset_flags()
+
+        # Test flashcopy_rate out of range
         spec = {'flashcopy_rate': 151}
         type_ref = volume_types.create(self.ctxt, "fccopy_rate", spec)
-        vol2 = self._generate_vol_info(type_ref)
+        vol3 = self._generate_vol_info(type_ref)
-        self.driver.create_volume(vol2)
+        self.driver.create_volume(vol3)
-        snap2 = self._generate_snap_info(vol2.id)
+        snap3 = self._generate_snap_info(vol3.id)
         self.assertRaises(exception.InvalidInput,
-                          self.driver.create_snapshot, snap2)
+                          self.driver.create_snapshot, snap3)
-        self._assert_vol_exists(snap2['name'], False)
+        self._assert_vol_exists(snap3['name'], False)
 
         # Test prestartfcmap failing
         with mock.patch.object(
@@ -5243,6 +5253,9 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         vol3 = testutils.create_volume(
             self.ctxt,
             volume_type_id=self.vt['id'])
+        vol4 = testutils.create_volume(
+            self.ctxt,
+            volume_type_id=self.vt['id'])
 
         # Try to clone where source size = target size
         vol1['size'] = vol2['size']
@@ -5268,7 +5281,20 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
                     self.assertEqual('49', fcmap['copyrate'])
         self._assert_vol_exists(vol3['name'], True)
 
+        # Try to clone and check if clean_rate is set to default
+        if self.USESIM:
+            self.sim.error_injection('lsfcmap', 'speed_up')
+        self.driver.create_cloned_volume(vol4, vol1)
+        if self.USESIM:
+            # Validate copyrate was set on the flash copy
+            for i, fcmap in self.sim._fcmappings_list.items():
+                if fcmap['target'] == vol1['name']:
+                    self.assertEqual('50', fcmap['cleanrate'])
+        self._assert_vol_exists(vol4['name'], True)
+
         # Delete in the 'opposite' order to make sure it works
+        self.driver.delete_volume(vol4)
+        self._assert_vol_exists(vol4['name'], False)
         self.driver.delete_volume(vol3)
         self._assert_vol_exists(vol3['name'], False)
         self.driver.delete_volume(vol2)
@@ -5327,6 +5353,51 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         attributes = self.driver._helpers.get_vdisk_attributes(volume4['name'])
         self.assertEqual('1', attributes['IO_group_id'])
 
+    def test_storwize_svc_retype_only_change_clean_rate(self):
+        self.driver.do_setup(None)
+        loc = ('StorwizeSVCDriver:' + self.driver._state['system_id'] +
+               ':openstack')
+        cap = {'location_info': loc, 'extent_size': '128'}
+        self.driver._stats = {'location_info': loc}
+        host = {'host': 'openstack@svc#openstack', 'capabilities': cap}
+        ctxt = context.get_admin_context()
+
+        key_specs_old = {'clean_rate': 50}
+        key_specs_new = {'clean_rate': 100}
+        old_type_ref = volume_types.create(ctxt, 'old', key_specs_old)
+        new_type_ref = volume_types.create(ctxt, 'new', key_specs_new)
+        host = {'host': 'openstack@svc#openstack'}
+        diff, _equal = volume_types.volume_types_diff(ctxt, old_type_ref['id'],
+                                                      new_type_ref['id'])
+
+        old_type = objects.VolumeType.get_by_id(ctxt,
+                                                old_type_ref['id'])
+        volume = self._generate_vol_info(old_type)
+        volume['host'] = host['host']
+        new_type = objects.VolumeType.get_by_id(ctxt,
+                                                new_type_ref['id'])
+
+        self.driver.create_volume(volume)
+        volume2 = testutils.create_volume(
+            self.ctxt,
+            volume_type_id=self.vt['id'])
+        self.driver.retype(ctxt, volume, new_type, diff, host)
+        if self.USESIM:
+            self.sim.error_injection('lsfcmap', 'speed_up')
+        self.driver.create_cloned_volume(volume2, volume)
+        if self.USESIM:
+            # Validate cleanrate was set on the flash copy
+            for i, fcmap in self.sim._fcmappings_list.items():
+                if fcmap['target'] == volume['name']:
+                    self.assertEqual('100', fcmap['cleanrate'])
+        self._assert_vol_exists(volume2['name'], True)
+
+        # Delete the volumes
+        self.driver.delete_volume(volume2)
+        self._assert_vol_exists(volume2['name'], False)
+        self.driver.delete_volume(volume)
+        self._assert_vol_exists(volume['name'], False)
+
     def test_storwize_svc_create_volume_from_snapshot(self):
         vol1 = self._create_volume()
         snap1 = self._generate_snap_info(vol1.id)
@@ -6869,7 +6940,8 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
         config = self.driver.configuration
         pool = "openstack2"
         if empty_qos:
-            opts = {'rsize': 2, 'iogrp': 0, 'qos': None, 'flashcopy_rate': 50}
+            opts = {'rsize': 2, 'iogrp': 0, 'qos': None, 'flashcopy_rate': 50,
+                    'clean_rate': 50}
             self.driver._helpers.create_flashcopy_to_consistgrp(
                 source, target, consistgrp, config,
                 opts, full_copy=False, pool=pool)
@@ -6878,7 +6950,8 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
 
         qos = {'IOThrottling': fake_iothrottling_value,
                'IOThrottling_unit': fake_iothrottling_unit}
-        opts = {'rsize': 2, 'iogrp': 0, 'qos': qos, 'flashcopy_rate': 50}
+        opts = {'rsize': 2, 'iogrp': 0, 'qos': qos, 'flashcopy_rate': 50,
+                'clean_rate': 50}
         self.driver._helpers.create_flashcopy_to_consistgrp(source,
                                                             target, consistgrp,
                                                             config, opts,
@@ -7737,7 +7810,9 @@ class StorwizeSVCCommonDriverTestCase(test.TestCase):
             self.driver.revert_to_snapshot(self.ctxt, vol2, snap2)
             mkfcmap.assert_called_once_with(snap2.name, vol2.name, True,
                                             self.driver.configuration.
-                                            storwize_svc_flashcopy_rate)
+                                            storwize_svc_flashcopy_rate,
+                                            self.driver.configuration.
+                                            storwize_svc_clean_rate)
             prepare_fc_map.assert_called_once_with(
                 '1', self.driver.configuration.storwize_svc_flashcopy_timeout,
                 True)
@@ -10135,6 +10210,18 @@ class StorwizeHelpersTestCase(test.TestCase):
                               self.storwize_svc_common.check_flashcopy_rate,
                               flashcopy_rate)
 
+    @mock.patch.object(storwize_svc_common.StorwizeSSH, 'chfcmap')
+    @mock.patch.object(storwize_svc_common.StorwizeHelpers,
+                       '_get_vdisk_fc_mappings')
+    def test_storwize_update_clean_rate(self,
+                                        chfcmap,
+                                        get_vdisk_fc_mappings):
+        get_vdisk_fc_mappings.return_value = ['4']
+        vol = 'test_vol'
+        new_clean_rate = 50
+        self.storwize_svc_common.update_clean_rate(vol, new_clean_rate)
+        chfcmap.assert_called()
+
     @ddt.data(({'mirror_pool': 'openstack2',
                 'volume_topology': None,
                 'peer_pool': None}, True, 1),
@@ -11495,7 +11582,8 @@ class StorwizeSVCReplicationTestCase(test.TestCase):
             self.driver.revert_to_snapshot(self.ctxt, vol1, snap1)
             mkfcmap.assert_called_once_with(
                 snap1.name, vol1.name, True,
-                self.driver.configuration.storwize_svc_flashcopy_rate)
+                self.driver.configuration.storwize_svc_flashcopy_rate,
+                self.driver.configuration.storwize_svc_clean_rate)
             prepare_fc_map.assert_called_once_with(
                 '1', self.driver.configuration.storwize_svc_flashcopy_timeout,
                 True)

cinder/volume/drivers/ibm/storwize_svc/storwize_svc_common.py

@@ -123,6 +123,12 @@ storwize_svc_opts = [
                help='Specifies the Storwize FlashCopy copy rate to be used '
                'when creating a full volume copy. The default is rate '
                'is 50, and the valid rates are 1-150.'),
+    cfg.IntOpt('storwize_svc_clean_rate',
+               default=50,
+               min=0, max=150,
+               help='Specifies the Storwize cleaning rate for the mapping. '
+                    'The default rate is 50, and the valid rates are '
+                    '0-150.'),
     cfg.StrOpt('storwize_svc_mirror_pool',
                default=None,
                help='Specifies the name of the pool in which mirrored copy '
@@ -596,7 +602,8 @@ class StorwizeSSH(object):
              '-unit', 'gb', '"%s"' % vdisk])
         self.run_ssh_assert_no_output(ssh_cmd)
 
-    def mkfcmap(self, source, target, full_copy, copy_rate, consistgrp=None):
+    def mkfcmap(self, source, target, full_copy, copy_rate, clean_rate,
+                consistgrp=None):
         ssh_cmd = ['svctask', 'mkfcmap', '-source', '"%s"' % source, '-target',
                    '"%s"' % target]
         if not full_copy:
@@ -606,6 +613,8 @@ class StorwizeSSH(object):
             ssh_cmd.append('-autodelete')
         if consistgrp:
             ssh_cmd.extend(['-consistgrp', consistgrp])
+        if clean_rate is not None:
+            ssh_cmd.extend(['-cleanrate', str(int(clean_rate))])
         out, err = self._ssh(ssh_cmd, check_exit_code=False)
         if 'successfully created' not in out:
             msg = (_('CLI Exception output:\n command: %(cmd)s\n '
@@ -655,9 +664,14 @@ class StorwizeSSH(object):
         ssh_cmd = ['svctask', 'stopfcconsistgrp', fc_consist_group]
         self.run_ssh_assert_no_output(ssh_cmd)
 
-    def chfcmap(self, fc_map_id, copyrate='50', autodel='on'):
+    def chfcmap(self, fc_map_id, copyrate=None, clean_rate=None,
-        ssh_cmd = ['svctask', 'chfcmap', '-copyrate', copyrate,
+                autodel='on'):
-                   '-autodelete', autodel, fc_map_id]
+        ssh_cmd = ['svctask', 'chfcmap']
+        if clean_rate is not None:
+            ssh_cmd += ['-cleanrate', clean_rate]
+        if copyrate is not None:
+            ssh_cmd += ['-copyrate', copyrate]
+        ssh_cmd += ['-autodelete', autodel, fc_map_id]
         self.run_ssh_assert_no_output(ssh_cmd)
 
     def stopfcmap(self, fc_map_id, force=False, split=False):
@@ -1496,6 +1510,7 @@ class StorwizeHelpers(object):
                'replication': False,
                'nofmtdisk': config.storwize_svc_vol_nofmtdisk,
                'flashcopy_rate': config.storwize_svc_flashcopy_rate,
+               'clean_rate': config.storwize_svc_clean_rate,
                'mirror_pool': config.storwize_svc_mirror_pool,
                'volume_topology': None,
                'peer_pool': config.storwize_peer_pool,
@@ -2218,6 +2233,12 @@ class StorwizeHelpers(object):
                      {'cg': cgId})
         return volume_model_updates
 
+    def update_clean_rate(self, volume_name, new_clean_rate):
+        mapping_ids = self._get_vdisk_fc_mappings(volume_name)
+        for map_id in mapping_ids:
+            self.ssh.chfcmap(map_id,
+                             clean_rate=six.text_type(new_clean_rate))
+
     def check_flashcopy_rate(self, flashcopy_rate):
         if not self.code_level:
             sys_info = self.get_system_info()
@@ -2247,13 +2268,14 @@ class StorwizeHelpers(object):
                                  copyrate=six.text_type(new_flashcopy_rate))
 
     def run_flashcopy(self, source, target, timeout, copy_rate,
-                      full_copy=True, restore=False):
+                      clean_rate, full_copy=True, restore=False):
         """Create a FlashCopy mapping from the source to the target."""
         LOG.debug('Enter: run_flashcopy: execute FlashCopy from source '
                   '%(source)s to target %(target)s.',
                   {'source': source, 'target': target})
         self.check_flashcopy_rate(copy_rate)
-        fc_map_id = self.ssh.mkfcmap(source, target, full_copy, copy_rate)
+        fc_map_id = self.ssh.mkfcmap(source, target, full_copy, copy_rate,
+                                     clean_rate)
         self._prepare_fc_map(fc_map_id, timeout, restore)
         self.ssh.startfcmap(fc_map_id, restore)
 
@@ -2293,6 +2315,7 @@ class StorwizeHelpers(object):
         self.check_flashcopy_rate(opts['flashcopy_rate'])
         self.ssh.mkfcmap(source, target, full_copy,
                          opts['flashcopy_rate'],
+                         opts['clean_rate'],
                          consistgrp=consistgrp)
 
         LOG.debug('Leave: create_flashcopy_to_consistgrp: '
@@ -2698,6 +2721,7 @@ class StorwizeHelpers(object):
         try:
             self.run_flashcopy(src, tgt, timeout,
                                opts['flashcopy_rate'],
+                               opts['clean_rate'],
                                full_copy=full_copy)
         except Exception:
             with excutils.save_and_reraise_exception():
@@ -3901,7 +3925,7 @@ class StorwizeSVCCommonDriver(san.SanDriver,
                 hs_opts = self._get_vdisk_params(volume['volume_type_id'],
                                                  volume_metadata=
                                                  volume.get(
-                                                     'volume_matadata'))
+                                                     'volume_metadata'))
                 try:
                     master_helper.convert_hyperswap_volume_to_normal(
                         volume_name, hs_opts['peer_pool'])
@@ -3990,7 +4014,7 @@ class StorwizeSVCCommonDriver(san.SanDriver,
         # Update the QoS IOThrottling value to the volume properties
         opts = self._get_vdisk_params(volume['volume_type_id'],
                                       volume_metadata=
-                                      volume.get('volume_matadata'))
+                                      volume.get('volume_metadata'))
         if opts['qos'] and opts['qos']['IOThrottling_unit']:
             unit = opts['qos']['IOThrottling_unit']
             if storwize_const.IOPS_PER_GB in unit:
@@ -5370,7 +5394,7 @@ class StorwizeSVCCommonDriver(san.SanDriver,
         all_keys = no_copy_keys + copy_keys
         old_opts = self._get_vdisk_params(volume['volume_type_id'],
                                           volume_metadata=
-                                          volume.get('volume_matadata'))
+                                          volume.get('volume_metadata'))
         new_opts = self._get_vdisk_params(new_type['id'],
                                           volume_type=new_type)
 
@@ -5487,6 +5511,12 @@ class StorwizeSVCCommonDriver(san.SanDriver,
             self._helpers.update_flashcopy_rate(volume.name,
                                                 new_opts['flashcopy_rate'])
 
+        if new_opts['clean_rate']:
+            # Add the new clean_rate. If the old FC maps has the clean_rate
+            # it will be overwritten.
+            self._helpers.update_clean_rate(volume.name,
+                                            new_opts['clean_rate'])
+
         # Delete replica if needed
         if self._state['code_level'] < (7, 7, 0, 0):
             force_unmap = False
@@ -6212,7 +6242,7 @@ class StorwizeSVCCommonDriver(san.SanDriver,
             self._helpers.run_flashcopy(
                 snapshot.name, volume.name,
                 self.configuration.storwize_svc_flashcopy_timeout,
-                opts['flashcopy_rate'], True, True)
+                opts['flashcopy_rate'], opts['clean_rate'], True, True)
             if rep_type:
                 self._helpers.start_relationship(volume.name, primary=None)
         except Exception as err:

doc/source/configuration/block-storage/drivers/ibm-storwize-svc-driver.rst

@@ -341,6 +341,7 @@ Family driver:
 - volume_topology
 - peer_pool
 - flashcopy_rate
+- clean_rate
 - cycle_period_seconds
 
 These keys have the same semantics as their counterparts in the

doc/source/configuration/tables/cinder-storwize.inc

@@ -30,6 +30,8 @@
      - (Boolean) Allow tenants to specify QoS on create.
    * - ``storwize_svc_flashcopy_rate`` = ``50``
      - (Integer) Specifies the Spectrum Virtualize Family FlashCopy copy rate to be used when creating a full volume copy. The default is rate is 50, and the valid rates are 1-100.
+   * - ``storwize_svc_clean_rate`` = ``50``
+     - (Integer) Specifies the Storwize cleaning rate for the mapping. The default rate is 50, and the valid rates are 0-150.
    * - ``storwize_svc_flashcopy_timeout`` = ``120``
      - (Integer) Maximum number of seconds to wait for FlashCopy to be prepared.
    * - ``storwize_svc_iscsi_chap_enabled`` = ``True``

releasenotes/notes/ibm-svf-add-cleanrate-support-e246a8f218d2f22e.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    IBM Spectrum Virtualize Family driver: Added support for clean_rate
+    parameter. Clean_rate parameter can now be passed as extra-spec in
+    volume-type or fetched from cinder.conf.