diff --git a/manila/tests/test_utils.py b/manila/tests/test_utils.py index 106ec41b..a3c419f9 100644 --- a/manila/tests/test_utils.py +++ b/manila/tests/test_utils.py @@ -21,6 +21,7 @@ import os import os.path import socket import tempfile +import time import uuid import ddt @@ -597,3 +598,87 @@ class IsValidIPVersion(test.TestCase): def test_provided_invalid_v4_address(self, addr): for vers in (4, '4'): self.assertFalse(utils.is_valid_ip_address(addr, vers)) + + +class TestRetryDecorator(test.TestCase): + def setUp(self): + super(TestRetryDecorator, self).setUp() + + def test_no_retry_required(self): + self.counter = 0 + + with mock.patch.object(time, 'sleep') as mock_sleep: + @utils.retry(exception.ManilaException, + interval=2, + retries=3, + backoff_rate=2) + def succeeds(): + self.counter += 1 + return 'success' + + ret = succeeds() + self.assertFalse(mock_sleep.called) + self.assertEqual('success', ret) + self.assertEqual(1, self.counter) + + def test_retries_once(self): + self.counter = 0 + interval = 2 + backoff_rate = 2 + retries = 3 + + with mock.patch.object(time, 'sleep') as mock_sleep: + @utils.retry(exception.ManilaException, + interval, + retries, + backoff_rate) + def fails_once(): + self.counter += 1 + if self.counter < 2: + raise exception.ManilaException(data='fake') + else: + return 'success' + + ret = fails_once() + self.assertEqual('success', ret) + self.assertEqual(2, self.counter) + self.assertEqual(1, mock_sleep.call_count) + mock_sleep.assert_called_with(interval * backoff_rate) + + def test_limit_is_reached(self): + self.counter = 0 + retries = 3 + interval = 2 + backoff_rate = 4 + + with mock.patch.object(time, 'sleep') as mock_sleep: + @utils.retry(exception.ManilaException, + interval, + retries, + backoff_rate) + def always_fails(): + self.counter += 1 + raise exception.ManilaException(data='fake') + + self.assertRaises(exception.ManilaException, + always_fails) + self.assertEqual(retries, self.counter) + + expected_sleep_arg = [] + + for i in range(retries): + if i > 0: + interval *= backoff_rate + expected_sleep_arg.append(float(interval)) + + mock_sleep.assert_has_calls(map(mock.call, expected_sleep_arg)) + + def test_wrong_exception_no_retry(self): + + with mock.patch.object(time, 'sleep') as mock_sleep: + @utils.retry(exception.ManilaException) + def raise_unexpected_error(): + raise ValueError("value error") + + self.assertRaises(ValueError, raise_unexpected_error) + self.assertFalse(mock_sleep.called) diff --git a/manila/utils.py b/manila/utils.py index 701e74dd..93fd4acb 100644 --- a/manila/utils.py +++ b/manila/utils.py @@ -36,6 +36,7 @@ from oslo_log import log from oslo_utils import importutils from oslo_utils import timeutils import paramiko +import retrying import six from manila.db import api as db_api @@ -466,3 +467,56 @@ class IsAMatcher(object): def __eq__(self, actual_value): return isinstance(actual_value, self.expected_value) + + +def retry(exception, interval=1, retries=10, backoff_rate=2): + """A wrapper around retrying library. + + This decorator allows to log and to check 'retries' input param. + Time interval between retries is calculated in the following way: + interval * backoff_rate ^ previous_attempt_number + + :param exception: expected exception type. When wrapped function + raises an exception of this type,the function + execution is retried. + :param interval: param 'interval' is used to calculate time interval + between retries: + interval * backoff_rate ^ previous_attempt_number + :param retries: number of retries + :param backoff_rate: param 'backoff_rate' is used to calculate time + interval between retries: + interval * backoff_rate ^ previous_attempt_number + + """ + def _retry_on_exception(e): + return isinstance(e, exception) + + def _backoff_sleep(previous_attempt_number, delay_since_first_attempt_ms): + exp = backoff_rate ** previous_attempt_number + wait_for = max(0, interval * exp) + LOG.debug("Sleeping for %s seconds", wait_for) + return wait_for * 1000.0 + + def _print_stop(previous_attempt_number, delay_since_first_attempt_ms): + delay_since_first_attempt = delay_since_first_attempt_ms / 1000.0 + LOG.debug("Failed attempt %s", previous_attempt_number) + LOG.debug("Have been at this for %s seconds", + delay_since_first_attempt) + return previous_attempt_number == retries + + if retries < 1: + raise ValueError(_('Retries must be greater than or ' + 'equal to 1 (received: %s).') % retries) + + def _decorator(f): + + @six.wraps(f) + def _wrapper(*args, **kwargs): + r = retrying.Retrying(retry_on_exception=_retry_on_exception, + wait_func=_backoff_sleep, + stop_func=_print_stop) + return r.call(f, *args, **kwargs) + + return _wrapper + + return _decorator