From 229515178494688f70f57d5ec29629ee3377373d Mon Sep 17 00:00:00 2001
From: Alyson Rosa <alyson.rodrigues.rosa@gmail.com>
Date: Mon, 30 Jan 2017 10:27:53 -0200
Subject: [PATCH] HNAS: Fix concurrency error when managing snapshots

During manage snapshot operation, if HNAS is already running a command
'path-to-object-number' it may cause concurrency issues, since HNAS
can run only one 'path-to-object-number' command at a time.

Fixing it by adding a retry when this message is returned.

Change-Id: If0f0b2d6f7e4ba3203d10c549181a3d31113624e
Closes-bug: #1660288
---
 manila/exception.py                           |  4 ++++
 manila/share/drivers/hitachi/hnas/ssh.py      |  6 ++++++
 .../share/drivers/hitachi/hnas/test_ssh.py    | 20 +++++++++++++++++++
 ...anage_snapshots_hnas-2c0e1a47b5e6ac33.yaml |  4 ++++
 4 files changed, 34 insertions(+)
 create mode 100644 releasenotes/notes/fix_manage_snapshots_hnas-2c0e1a47b5e6ac33.yaml

diff --git a/manila/exception.py b/manila/exception.py
index d0b813b3e8..b0a537dc64 100644
--- a/manila/exception.py
+++ b/manila/exception.py
@@ -812,6 +812,10 @@ class HNASConnException(ManilaException):
     message = _("HNAS Connection Exception: %(msg)s")
 
 
+class HNASSSCIsBusy(ManilaException):
+    message = _("HNAS SSC is busy and cannot execute the command: %(msg)s")
+
+
 class HNASItemNotFoundException(StorageResourceNotFound):
     message = _("HNAS Item Not Found Exception: %(msg)s")
 
diff --git a/manila/share/drivers/hitachi/hnas/ssh.py b/manila/share/drivers/hitachi/hnas/ssh.py
index b96ad345fa..1cc69dd118 100644
--- a/manila/share/drivers/hitachi/hnas/ssh.py
+++ b/manila/share/drivers/hitachi/hnas/ssh.py
@@ -309,12 +309,18 @@ class HNASSSHBackend(object):
     def delete_directory(self, path):
         self._locked_selectfs('delete', path)
 
+    @mutils.retry(exception=exception.HNASSSCIsBusy, wait_random=True,
+                  retries=5)
     def check_snapshot(self, path):
         command = ['path-to-object-number', '-f', self.fs_name, path]
 
         try:
             self._execute(command)
         except processutils.ProcessExecutionError as e:
+            if 'path-to-object-number is currently running' in e.stdout:
+                msg = (_("SSC command path-to-object-number for path %s "
+                         "is currently busy.") % path)
+                raise exception.HNASSSCIsBusy(msg=msg)
             if 'Unable to locate component:' in e.stdout:
                 LOG.debug("Cannot find %(path)s: %(out)s",
                           {'path': path, 'out': e.stdout})
diff --git a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
index 255bc419d7..48d6881eb6 100644
--- a/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
+++ b/manila/tests/share/drivers/hitachi/hnas/test_ssh.py
@@ -936,6 +936,26 @@ class HNASSSHTestCase(test.TestCase):
         self.assertTrue(out)
         self._driver_ssh._execute.assert_called_with(check_snap_args)
 
+    def test_check_snapshot_retry(self):
+        error_msg = ("Unable to run path-to-object-number as "
+                     "path-to-object-number is currently running on volume "
+                     "39.")
+        path = ("/snapshots/" + self.snapshot['share_id'] + "/" +
+                self.snapshot['id'])
+
+        check_snap_args = ['path-to-object-number', '-f', self.fs_name, path]
+
+        self.mock_object(time, "sleep", mock.Mock())
+        self.mock_object(ssh.HNASSSHBackend, '_execute',
+                         mock.Mock(side_effect=[putils.ProcessExecutionError(
+                             stdout=error_msg), putils.ProcessExecutionError(
+                             stdout=error_msg), 'Object number: 0x45a4']))
+
+        out = self._driver_ssh.check_snapshot(path)
+
+        self.assertIs(True, out)
+        self._driver_ssh._execute.assert_called_with(check_snap_args)
+
     def test_check_inexistent_snapshot(self):
         path = "/path/snap1/snapshot07-08-2016"
 
diff --git a/releasenotes/notes/fix_manage_snapshots_hnas-2c0e1a47b5e6ac33.yaml b/releasenotes/notes/fix_manage_snapshots_hnas-2c0e1a47b5e6ac33.yaml
new file mode 100644
index 0000000000..0e7cec1079
--- /dev/null
+++ b/releasenotes/notes/fix_manage_snapshots_hnas-2c0e1a47b5e6ac33.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - Fixed HNAS driver error when managing snapshots caused by
+    concurrency in backend.