From cf637479c9b2356e6b011c6b000c68561a1b045a Mon Sep 17 00:00:00 2001
From: Julia Varlamova <jvarlamova@mirantis.com>
Date: Fri, 10 Jul 2015 09:20:29 -0400
Subject: [PATCH] Retry _unmount_device in generic driver

We get 'umount: Device is busy ' error on delete,
extend and shrink operations in generic driver.

Add 'retry' decorator to _unmount_device function in generic driver
to allow driver to wait until unmounting will be possible.

Closes-bug: #1469068

Change-Id: I2192ed76978cd37051c8b71528ad5dcd3c222d67
---
 manila/share/drivers/generic.py            |  1 +
 manila/tests/share/drivers/test_generic.py | 36 ++++++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/manila/share/drivers/generic.py b/manila/share/drivers/generic.py
index 38ef81d6fa..a95ec4326b 100644
--- a/manila/share/drivers/generic.py
+++ b/manila/share/drivers/generic.py
@@ -297,6 +297,7 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
                 raise exception.ShareBackendException(msg=six.text_type(e))
         return _mount_device_with_lock()
 
+    @utils.retry(exception.ProcessExecutionError)
     def _unmount_device(self, share, server_details):
         """Unmounts block device from directory on service vm."""
 
diff --git a/manila/tests/share/drivers/test_generic.py b/manila/tests/share/drivers/test_generic.py
index 4e5afb6ca1..bea48b60d6 100644
--- a/manila/tests/share/drivers/test_generic.py
+++ b/manila/tests/share/drivers/test_generic.py
@@ -17,6 +17,7 @@
 """Unit tests for the Generic driver module."""
 
 import os
+import time
 
 import ddt
 import mock
@@ -279,6 +280,41 @@ class GenericShareDriverTestCase(test.TestCase):
             ['sudo umount', mount_path, '&& sudo rmdir', mount_path],
         )
 
+    def test_unmount_device_retry_once(self):
+        self.counter = 0
+
+        def _side_effect(*args):
+            self.counter += 1
+            if self.counter < 2:
+                raise exception.ProcessExecutionError
+
+        mount_path = '/fake/mount/path'
+        self.mock_object(self._driver, '_is_device_mounted',
+                         mock.Mock(return_value=True))
+        self.mock_object(self._driver, '_sync_mount_temp_and_perm_files')
+        self.mock_object(self._driver, '_get_mount_path',
+                         mock.Mock(return_value=mount_path))
+        self.mock_object(self._driver, '_ssh_exec',
+                         mock.Mock(side_effect=_side_effect))
+        self.mock_object(time, 'sleep')
+
+        self._driver._unmount_device(self.share, self.server)
+
+        self.assertEqual(1, time.sleep.call_count)
+        self.assertEqual(self._driver._get_mount_path.mock_calls,
+                         [mock.call(self.share) for i in xrange(2)])
+        self.assertEqual(self._driver._is_device_mounted.mock_calls,
+                         [mock.call(mount_path,
+                                    self.server) for i in xrange(2)])
+        self._driver._sync_mount_temp_and_perm_files.assert_called_once_with(
+            self.server)
+        self.assertEqual(
+            self._driver._ssh_exec.mock_calls,
+            [mock.call(self.server, ['sudo umount', mount_path,
+                                     '&& sudo rmdir', mount_path])
+             for i in xrange(2)]
+        )
+
     def test_unmount_device_not_present(self):
         mount_path = '/fake/mount/path'
         self.mock_object(self._driver, '_is_device_mounted',