Add 'retry' wrapper to manila/utils.py

Retrying library from OpenStack Requirement is already used in Manila
project. Add 'retry' wrapper around it which enables logging and
checking of input param 'retries' which may be useful.

Related-Bug: #1469068

Change-Id: I159b6e44fec9455634d76c093036f934cf400366

manila/tests/test_utils.py

@@ -21,6 +21,7 @@ import os
 import os.path
 import socket
 import tempfile
+import time
 import uuid
 
 import ddt
@@ -597,3 +598,87 @@ class IsValidIPVersion(test.TestCase):
     def test_provided_invalid_v4_address(self, addr):
         for vers in (4, '4'):
             self.assertFalse(utils.is_valid_ip_address(addr, vers))
+
+
+class TestRetryDecorator(test.TestCase):
+    def setUp(self):
+        super(TestRetryDecorator, self).setUp()
+
+    def test_no_retry_required(self):
+        self.counter = 0
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.ManilaException,
+                         interval=2,
+                         retries=3,
+                         backoff_rate=2)
+            def succeeds():
+                self.counter += 1
+                return 'success'
+
+            ret = succeeds()
+            self.assertFalse(mock_sleep.called)
+            self.assertEqual('success', ret)
+            self.assertEqual(1, self.counter)
+
+    def test_retries_once(self):
+        self.counter = 0
+        interval = 2
+        backoff_rate = 2
+        retries = 3
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.ManilaException,
+                         interval,
+                         retries,
+                         backoff_rate)
+            def fails_once():
+                self.counter += 1
+                if self.counter < 2:
+                    raise exception.ManilaException(data='fake')
+                else:
+                    return 'success'
+
+            ret = fails_once()
+            self.assertEqual('success', ret)
+            self.assertEqual(2, self.counter)
+            self.assertEqual(1, mock_sleep.call_count)
+            mock_sleep.assert_called_with(interval * backoff_rate)
+
+    def test_limit_is_reached(self):
+        self.counter = 0
+        retries = 3
+        interval = 2
+        backoff_rate = 4
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.ManilaException,
+                         interval,
+                         retries,
+                         backoff_rate)
+            def always_fails():
+                self.counter += 1
+                raise exception.ManilaException(data='fake')
+
+            self.assertRaises(exception.ManilaException,
+                              always_fails)
+            self.assertEqual(retries, self.counter)
+
+            expected_sleep_arg = []
+
+            for i in range(retries):
+                if i > 0:
+                    interval *= backoff_rate
+                    expected_sleep_arg.append(float(interval))
+
+            mock_sleep.assert_has_calls(map(mock.call, expected_sleep_arg))
+
+    def test_wrong_exception_no_retry(self):
+
+        with mock.patch.object(time, 'sleep') as mock_sleep:
+            @utils.retry(exception.ManilaException)
+            def raise_unexpected_error():
+                raise ValueError("value error")
+
+            self.assertRaises(ValueError, raise_unexpected_error)
+            self.assertFalse(mock_sleep.called)

manila/utils.py

@@ -36,6 +36,7 @@ from oslo_log import log
 from oslo_utils import importutils
 from oslo_utils import timeutils
 import paramiko
+import retrying
 import six
 
 from manila.db import api as db_api
@@ -466,3 +467,56 @@ class IsAMatcher(object):
 
     def __eq__(self, actual_value):
         return isinstance(actual_value, self.expected_value)
+
+
+def retry(exception, interval=1, retries=10, backoff_rate=2):
+    """A wrapper around retrying library.
+
+        This decorator allows to log and to check 'retries' input param.
+        Time interval between retries is calculated in the following way:
+        interval * backoff_rate ^ previous_attempt_number
+
+        :param exception: expected exception type. When wrapped function
+                          raises an exception of this type,the function
+                          execution is retried.
+        :param interval: param 'interval' is used to calculate time interval
+                         between retries:
+                         interval * backoff_rate ^ previous_attempt_number
+        :param retries: number of retries
+        :param backoff_rate: param 'backoff_rate' is used to calculate time
+                             interval between retries:
+                             interval * backoff_rate ^ previous_attempt_number
+
+    """
+    def _retry_on_exception(e):
+        return isinstance(e, exception)
+
+    def _backoff_sleep(previous_attempt_number, delay_since_first_attempt_ms):
+        exp = backoff_rate ** previous_attempt_number
+        wait_for = max(0, interval * exp)
+        LOG.debug("Sleeping for %s seconds", wait_for)
+        return wait_for * 1000.0
+
+    def _print_stop(previous_attempt_number, delay_since_first_attempt_ms):
+        delay_since_first_attempt = delay_since_first_attempt_ms / 1000.0
+        LOG.debug("Failed attempt %s", previous_attempt_number)
+        LOG.debug("Have been at this for %s seconds",
+                  delay_since_first_attempt)
+        return previous_attempt_number == retries
+
+    if retries < 1:
+        raise ValueError(_('Retries must be greater than or '
+                           'equal to 1 (received: %s).') % retries)
+
+    def _decorator(f):
+
+        @six.wraps(f)
+        def _wrapper(*args, **kwargs):
+            r = retrying.Retrying(retry_on_exception=_retry_on_exception,
+                                  wait_func=_backoff_sleep,
+                                  stop_func=_print_stop)
+            return r.call(f, *args, **kwargs)
+
+        return _wrapper
+
+    return _decorator