[NetApp] Pause and resume clone split during rename snapshot

When snpashot are soft deleted i.e. they are renamed if delete fails,
sometime we face issue in rename operation. This is due to busy
snapshot clone split operations. So perform rename and then start clone
split.

Closes-bug: #2025641
Change-Id: I1093a610f31f5971bb23b9f89c64f5f129ac2fb9

manila/share/drivers/netapp/dataontap/client/client_cmode.py

@@ -3522,6 +3522,12 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
             msg_args = {'snapshot': snapshot_name, 'volume': volume_name}
             LOG.info(msg, msg_args)
 
+            # Snapshots are locked by clone(s), so split the clone(s)
+            snapshot_children = self.get_clone_children_for_snapshot(
+                volume_name, snapshot_name)
+            for snapshot_child in snapshot_children:
+                self.volume_clone_split_start(snapshot_child['name'])
+
     @na_utils.trace
     def prune_deleted_snapshots(self):
         """Deletes non-busy snapshots that were previously soft-deleted."""

manila/share/drivers/netapp/dataontap/client/client_cmode_rest.py

@@ -2034,6 +2034,25 @@ class NetAppRestClient(object):
                     f'/storage/volumes/{uuid}/snapshots/{snapshot_uuid}',
                     'delete')
 
+    @na_utils.trace
+    def soft_delete_snapshot(self, volume_name, snapshot_name):
+        """Deletes a volume snapshot, or renames it if delete fails."""
+        try:
+            self.delete_snapshot(volume_name, snapshot_name)
+        except netapp_api.NaApiError:
+            self.rename_snapshot(volume_name,
+                                 snapshot_name,
+                                 DELETED_PREFIX + snapshot_name)
+            msg = _('Soft-deleted snapshot %(snapshot)s on volume %(volume)s.')
+            msg_args = {'snapshot': snapshot_name, 'volume': volume_name}
+            LOG.info(msg, msg_args)
+
+            # Snapshots are locked by clone(s), so split the clone(s)
+            snapshot_children = self.get_clone_children_for_snapshot(
+                volume_name, snapshot_name)
+            for snapshot_child in snapshot_children:
+                self.volume_clone_split_start(snapshot_child['name'])
+
     @na_utils.trace
     def rename_snapshot(self, volume_name, snapshot_name, new_snapshot_name):
         """Renames the snapshot."""

manila/share/drivers/netapp/dataontap/cluster_mode/lib_base.py

@@ -1883,13 +1883,14 @@ class NetAppCmodeFileStorageLibrary(object):
             vserver_client.delete_snapshot(share_name, snapshot_name)
 
         elif backend_snapshot['locked_by_clone']:
-            # Snapshots are locked by clone(s), so split the clone(s)
-            snapshot_children = vserver_client.get_clone_children_for_snapshot(
-                share_name, snapshot_name)
-            for snapshot_child in snapshot_children:
-                vserver_client.volume_clone_split_start(snapshot_child['name'])
-
             if is_flexgroup:
+                # Snapshots are locked by clone(s), so split the clone(s)
+                snap_children = vserver_client.get_clone_children_for_snapshot(
+                    share_name, snapshot_name)
+                for snapshot_child in snap_children:
+                    vserver_client.volume_clone_split_start(
+                        snapshot_child['name'])
+
                 # NOTE(felipe_rodrigues): ONTAP does not allow rename a
                 # FlexGroup snapshot, so it cannot be soft deleted. It will
                 # wait for all split clones complete.

manila/tests/share/drivers/netapp/dataontap/client/fakes.py

@@ -142,6 +142,13 @@ FPOLICY_EXT_TO_EXCLUDE_LIST = ['jpg', 'mp3']
 JOB_ID = 123
 JOB_STATE = 'success'
 
+CDOT_CLONE_CHILD_1 = 'fake_child_1'
+CDOT_CLONE_CHILD_2 = 'fake_child_2'
+CDOT_CLONE_CHILDREN = [
+    {'name': CDOT_CLONE_CHILD_1},
+    {'name': CDOT_CLONE_CHILD_2},
+]
+
 NETWORK_INTERFACES = [{
     'interface_name': 'fake_interface',
     'address': IP_ADDRESS,

manila/tests/share/drivers/netapp/dataontap/client/test_client_cmode.py

@@ -5116,6 +5116,11 @@ class NetAppClientCmodeTestCase(test.TestCase):
         mock_delete_snapshot = self.mock_object(
             self.client, 'delete_snapshot', self._mock_api_error())
         mock_rename_snapshot = self.mock_object(self.client, 'rename_snapshot')
+        mock_get_clone_children_for_snapshot = self.mock_object(
+            self.client, 'get_clone_children_for_snapshot',
+            mock.Mock(return_value=fake.CDOT_CLONE_CHILDREN))
+        mock_volume_clone_split_start = self.mock_object(
+            self.client, 'volume_clone_split_start')
 
         self.client.soft_delete_snapshot(fake.SHARE_NAME, fake.SNAPSHOT_NAME)
 
@@ -5124,6 +5129,12 @@ class NetAppClientCmodeTestCase(test.TestCase):
         mock_rename_snapshot.assert_called_once_with(
             fake.SHARE_NAME, fake.SNAPSHOT_NAME,
             'deleted_manila_' + fake.SNAPSHOT_NAME)
+        mock_get_clone_children_for_snapshot.assert_called_once_with(
+            fake.SHARE_NAME, fake.SNAPSHOT_NAME)
+        mock_volume_clone_split_start.assert_has_calls([
+            mock.call(fake.CDOT_CLONE_CHILD_1),
+            mock.call(fake.CDOT_CLONE_CHILD_2),
+        ])
 
     def test_prune_deleted_snapshots(self):
 

manila/tests/share/drivers/netapp/dataontap/client/test_client_cmode_rest.py

@@ -5941,6 +5941,17 @@ class NetAppRestCmodeClientTestCase(test.TestCase):
                                     ignore_owners)
         mock_sr.assert_has_calls(calls)
 
+    def test_soft_delete_snapshot(self):
+
+        mock_delete_snapshot = self.mock_object(self.client, 'delete_snapshot')
+        mock_rename_snapshot = self.mock_object(self.client, 'rename_snapshot')
+
+        self.client.soft_delete_snapshot(fake.SHARE_NAME, fake.SNAPSHOT_NAME)
+
+        mock_delete_snapshot.assert_called_once_with(
+            fake.SHARE_NAME, fake.SNAPSHOT_NAME)
+        self.assertFalse(mock_rename_snapshot.called)
+
     def test_volume_has_luns(self):
         mock_sr = self.mock_object(self.client, 'send_request')
         self.mock_object(self.client, '_has_records',

manila/tests/share/drivers/netapp/dataontap/cluster_mode/test_lib_base.py

@@ -2618,13 +2618,14 @@ class NetAppFileStorageLibraryTestCase(test.TestCase):
                                       is_flexgroup=is_flexgroup)
 
         self.assertFalse(vserver_client.delete_snapshot.called)
-        vserver_client.get_clone_children_for_snapshot.assert_called_once_with(
-            fake.SHARE_NAME, fake.SNAPSHOT_NAME)
-        vserver_client.volume_clone_split_start.assert_has_calls([
-            mock.call(fake.CDOT_CLONE_CHILD_1),
-            mock.call(fake.CDOT_CLONE_CHILD_2),
-        ])
         if is_flexgroup:
+            (vserver_client.
+                get_clone_children_for_snapshot.assert_called_once_with(
+                    fake.SHARE_NAME, fake.SNAPSHOT_NAME))
+            vserver_client.volume_clone_split_start.assert_has_calls([
+                mock.call(fake.CDOT_CLONE_CHILD_1),
+                mock.call(fake.CDOT_CLONE_CHILD_2),
+            ])
             mock_is_flexgroup_share.assert_called_once_with(
                 vserver_client, fake.SHARE_NAME, fake.SNAPSHOT_NAME)
             vserver_client.soft_delete_snapshot.assert_not_called()

releasenotes/notes/bug-2025641-pause-and-resume-clone-split-during-snapshot-rename-fd0f990d50644d9c.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    In case of NetApp ONTAP driver, when snpashots are soft deleted (i.e. they
+    are renamed if delete fails) sometime we face issue in during rename. This
+    is due to busy snapshots. To overcome this, Manila will stop clone split,
+    perform rename and start clone split again. For more details, please refer
+    to `launchpad bug 2025641 <https://bugs.launchpad.net/manila/+bug/2025641>`_