HNAS: Fix concurrency error when managing snapshots

During manage snapshot operation, if HNAS is already running a command
'path-to-object-number' it may cause concurrency issues, since HNAS
can run only one 'path-to-object-number' command at a time.

Fixing it by adding a retry when this message is returned.

Change-Id: If0f0b2d6f7e4ba3203d10c549181a3d31113624e
Closes-bug: #1660288

manila/exception.py

@@ -812,6 +812,10 @@ class HNASConnException(ManilaException):
     message = _("HNAS Connection Exception: %(msg)s")
 
 
+class HNASSSCIsBusy(ManilaException):
+    message = _("HNAS SSC is busy and cannot execute the command: %(msg)s")
+
+
 class HNASItemNotFoundException(StorageResourceNotFound):
     message = _("HNAS Item Not Found Exception: %(msg)s")
 

manila/share/drivers/hitachi/hnas/ssh.py

@@ -309,12 +309,18 @@ class HNASSSHBackend(object):
     def delete_directory(self, path):
         self._locked_selectfs('delete', path)
 
+    @mutils.retry(exception=exception.HNASSSCIsBusy, wait_random=True,
+                  retries=5)
     def check_snapshot(self, path):
         command = ['path-to-object-number', '-f', self.fs_name, path]
 
         try:
             self._execute(command)
         except processutils.ProcessExecutionError as e:
+            if 'path-to-object-number is currently running' in e.stdout:
+                msg = (_("SSC command path-to-object-number for path %s "
+                         "is currently busy.") % path)
+                raise exception.HNASSSCIsBusy(msg=msg)
             if 'Unable to locate component:' in e.stdout:
                 LOG.debug("Cannot find %(path)s: %(out)s",
                           {'path': path, 'out': e.stdout})

manila/tests/share/drivers/hitachi/hnas/test_ssh.py

@@ -936,6 +936,26 @@ class HNASSSHTestCase(test.TestCase):
         self.assertTrue(out)
         self._driver_ssh._execute.assert_called_with(check_snap_args)
 
+    def test_check_snapshot_retry(self):
+        error_msg = ("Unable to run path-to-object-number as "
+                     "path-to-object-number is currently running on volume "
+                     "39.")
+        path = ("/snapshots/" + self.snapshot['share_id'] + "/" +
+                self.snapshot['id'])
+
+        check_snap_args = ['path-to-object-number', '-f', self.fs_name, path]
+
+        self.mock_object(time, "sleep", mock.Mock())
+        self.mock_object(ssh.HNASSSHBackend, '_execute',
+                         mock.Mock(side_effect=[putils.ProcessExecutionError(
+                             stdout=error_msg), putils.ProcessExecutionError(
+                             stdout=error_msg), 'Object number: 0x45a4']))
+
+        out = self._driver_ssh.check_snapshot(path)
+
+        self.assertIs(True, out)
+        self._driver_ssh._execute.assert_called_with(check_snap_args)
+
     def test_check_inexistent_snapshot(self):
         path = "/path/snap1/snapshot07-08-2016"
 

releasenotes/notes/fix_manage_snapshots_hnas-2c0e1a47b5e6ac33.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - Fixed HNAS driver error when managing snapshots caused by
+    concurrency in backend.