Netapp: Retry Dedupe enable/disable if busy

With some NetApp hardware models, driver turn on efficiency by default.
That is why Manila might see the conflicting sis operation. Retry for 5
times, in case dedupe enable/disable failed with error "Another sis
operation is currently active."

Closes-bug: #2071359
Change-Id: I6229a98063d393289e2225ceacd84145ecf320d0

manila/share/drivers/netapp/dataontap/client/api.py

@@ -67,6 +67,7 @@ EPOLICYNOTFOUND = '18251'
 EEVENTNOTFOUND = '18253'
 ESCOPENOTFOUND = '18259'
 ESVMDR_CANNOT_PERFORM_OP_FOR_STATUS = '18815'
+OPERATION_ALREADY_ENABLED = '40043'
 ENFS_V4_0_ENABLED_MIGRATION_FAILURE = '13172940'
 EVSERVER_MIGRATION_TO_NON_AFF_CLUSTER = '13172984'
 

manila/share/drivers/netapp/dataontap/client/client_cmode.py

@@ -2327,16 +2327,50 @@ class NetAppCmodeClient(client_base.NetAppBaseClient):
         return api_args
 
     @na_utils.trace
+    @manila_utils.retry(retry_param=exception.NetAppException,
+                        interval=3,
+                        retries=5,
+                        backoff_rate=1)
     def enable_dedup(self, volume_name):
         """Enable deduplication on volume."""
         api_args = {'path': '/vol/%s' % volume_name}
-        self.send_request('sis-enable', api_args)
+        try:
+            self.send_request('sis-enable', api_args)
+            return
+        except netapp_api.NaApiError as e:
+            enabled_msg = "has already been enabled"
+            if (e.code == netapp_api.OPERATION_ALREADY_ENABLED and
+                    enabled_msg in e.message):
+                return
+            active_msg = "sis operation is currently active"
+            if (e.code == netapp_api.OPERATION_ALREADY_ENABLED and
+                    active_msg in e.message):
+                msg = _('Unable to enable dedup. Will retry the '
+                        'operation. Error details: %s') % e.message
+                LOG.warning(msg)
+                raise exception.NetAppException(msg=msg)
+            raise e
 
     @na_utils.trace
+    @manila_utils.retry(retry_param=exception.NetAppException,
+                        interval=3,
+                        retries=5,
+                        backoff_rate=1)
     def disable_dedup(self, volume_name):
         """Disable deduplication on volume."""
         api_args = {'path': '/vol/%s' % volume_name}
-        self.send_request('sis-disable', api_args)
+        try:
+            self.send_request('sis-disable', api_args)
+            return
+        except netapp_api.NaApiError as e:
+            active_msg = "sis operation is currently active"
+            if (e.code == netapp_api.OPERATION_ALREADY_ENABLED and
+                    active_msg in e.message):
+                msg = _('Unable to disable dedup. Will retry the '
+                        'operation. Error details: %s') % e.message
+                LOG.warning(msg)
+                raise exception.NetAppException(msg=msg)
+            raise e
 
     @na_utils.trace
     def enable_compression(self, volume_name):

manila/tests/share/drivers/netapp/dataontap/client/test_client_cmode.py

@@ -3501,6 +3501,35 @@ class NetAppClientCmodeTestCase(test.TestCase):
         self.client.send_request.assert_called_once_with('sis-enable',
                                                          sis_enable_args)
 
+    def test_enable_dedup_already_enabled(self):
+        side_effect = netapp_api.NaApiError(
+            code=netapp_api.OPERATION_ALREADY_ENABLED,
+            message='It has already been enabled')
+
+        self.mock_object(self.client,
+                         'send_request',
+                         mock.Mock(side_effect=side_effect))
+
+        self.client.enable_dedup(fake.SHARE_NAME)
+
+        sis_enable_args = {'path': '/vol/%s' % fake.SHARE_NAME}
+
+        self.client.send_request.assert_called_once_with('sis-enable',
+                                                         sis_enable_args)
+
+    def test_enable_dedup_currently_active(self):
+        side_effect = netapp_api.NaApiError(
+            code=netapp_api.OPERATION_ALREADY_ENABLED,
+            message='The sis operation is currently active')
+
+        self.mock_object(self.client,
+                         'send_request',
+                         mock.Mock(side_effect=side_effect))
+
+        self.assertRaises(exception.NetAppException,
+                          self.client.enable_dedup,
+                          fake.SHARE_NAME)
+
     def test_disable_dedup(self):
 
         self.mock_object(self.client, 'send_request')
@@ -3512,6 +3541,19 @@ class NetAppClientCmodeTestCase(test.TestCase):
         self.client.send_request.assert_called_once_with('sis-disable',
                                                          sis_disable_args)
 
+    def test_disable_dedup_currently_active(self):
+        side_effect = netapp_api.NaApiError(
+            code=netapp_api.OPERATION_ALREADY_ENABLED,
+            message='The sis operation is currently active')
+
+        self.mock_object(self.client,
+                         'send_request',
+                         mock.Mock(side_effect=side_effect))
+
+        self.assertRaises(exception.NetAppException,
+                          self.client.disable_dedup,
+                          fake.SHARE_NAME)
+
     def test_enable_compression(self):
 
         self.mock_object(self.client, 'send_request')

releasenotes/notes/bug-2071359-netapp-retry-sis-operatin-if-already-active-4625605175f76d07.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    NetApp ONTAP driver will now retry the sis operation (e.g. dedupe) if sis
+    operation is currently active. This is needed because NetApp turns on
+    efficiency (by default) on latest hardware which causes conflicting sis
+    operation when Manila tries to turn it off. For more details, please check
+    Launchpad `bug #2071359 <https://bugs.launchpad.net/manila/+bug/2071359>`_