Merge "Add 'retry' wrapper to manila/utils.py"
This commit is contained in:
commit
24c4026661
@ -21,6 +21,7 @@ import os
|
||||
import os.path
|
||||
import socket
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import ddt
|
||||
@ -597,3 +598,87 @@ class IsValidIPVersion(test.TestCase):
|
||||
def test_provided_invalid_v4_address(self, addr):
|
||||
for vers in (4, '4'):
|
||||
self.assertFalse(utils.is_valid_ip_address(addr, vers))
|
||||
|
||||
|
||||
class TestRetryDecorator(test.TestCase):
|
||||
def setUp(self):
|
||||
super(TestRetryDecorator, self).setUp()
|
||||
|
||||
def test_no_retry_required(self):
|
||||
self.counter = 0
|
||||
|
||||
with mock.patch.object(time, 'sleep') as mock_sleep:
|
||||
@utils.retry(exception.ManilaException,
|
||||
interval=2,
|
||||
retries=3,
|
||||
backoff_rate=2)
|
||||
def succeeds():
|
||||
self.counter += 1
|
||||
return 'success'
|
||||
|
||||
ret = succeeds()
|
||||
self.assertFalse(mock_sleep.called)
|
||||
self.assertEqual('success', ret)
|
||||
self.assertEqual(1, self.counter)
|
||||
|
||||
def test_retries_once(self):
|
||||
self.counter = 0
|
||||
interval = 2
|
||||
backoff_rate = 2
|
||||
retries = 3
|
||||
|
||||
with mock.patch.object(time, 'sleep') as mock_sleep:
|
||||
@utils.retry(exception.ManilaException,
|
||||
interval,
|
||||
retries,
|
||||
backoff_rate)
|
||||
def fails_once():
|
||||
self.counter += 1
|
||||
if self.counter < 2:
|
||||
raise exception.ManilaException(data='fake')
|
||||
else:
|
||||
return 'success'
|
||||
|
||||
ret = fails_once()
|
||||
self.assertEqual('success', ret)
|
||||
self.assertEqual(2, self.counter)
|
||||
self.assertEqual(1, mock_sleep.call_count)
|
||||
mock_sleep.assert_called_with(interval * backoff_rate)
|
||||
|
||||
def test_limit_is_reached(self):
|
||||
self.counter = 0
|
||||
retries = 3
|
||||
interval = 2
|
||||
backoff_rate = 4
|
||||
|
||||
with mock.patch.object(time, 'sleep') as mock_sleep:
|
||||
@utils.retry(exception.ManilaException,
|
||||
interval,
|
||||
retries,
|
||||
backoff_rate)
|
||||
def always_fails():
|
||||
self.counter += 1
|
||||
raise exception.ManilaException(data='fake')
|
||||
|
||||
self.assertRaises(exception.ManilaException,
|
||||
always_fails)
|
||||
self.assertEqual(retries, self.counter)
|
||||
|
||||
expected_sleep_arg = []
|
||||
|
||||
for i in range(retries):
|
||||
if i > 0:
|
||||
interval *= backoff_rate
|
||||
expected_sleep_arg.append(float(interval))
|
||||
|
||||
mock_sleep.assert_has_calls(map(mock.call, expected_sleep_arg))
|
||||
|
||||
def test_wrong_exception_no_retry(self):
|
||||
|
||||
with mock.patch.object(time, 'sleep') as mock_sleep:
|
||||
@utils.retry(exception.ManilaException)
|
||||
def raise_unexpected_error():
|
||||
raise ValueError("value error")
|
||||
|
||||
self.assertRaises(ValueError, raise_unexpected_error)
|
||||
self.assertFalse(mock_sleep.called)
|
||||
|
@ -36,6 +36,7 @@ from oslo_log import log
|
||||
from oslo_utils import importutils
|
||||
from oslo_utils import timeutils
|
||||
import paramiko
|
||||
import retrying
|
||||
import six
|
||||
|
||||
from manila.db import api as db_api
|
||||
@ -466,3 +467,56 @@ class IsAMatcher(object):
|
||||
|
||||
def __eq__(self, actual_value):
|
||||
return isinstance(actual_value, self.expected_value)
|
||||
|
||||
|
||||
def retry(exception, interval=1, retries=10, backoff_rate=2):
|
||||
"""A wrapper around retrying library.
|
||||
|
||||
This decorator allows to log and to check 'retries' input param.
|
||||
Time interval between retries is calculated in the following way:
|
||||
interval * backoff_rate ^ previous_attempt_number
|
||||
|
||||
:param exception: expected exception type. When wrapped function
|
||||
raises an exception of this type,the function
|
||||
execution is retried.
|
||||
:param interval: param 'interval' is used to calculate time interval
|
||||
between retries:
|
||||
interval * backoff_rate ^ previous_attempt_number
|
||||
:param retries: number of retries
|
||||
:param backoff_rate: param 'backoff_rate' is used to calculate time
|
||||
interval between retries:
|
||||
interval * backoff_rate ^ previous_attempt_number
|
||||
|
||||
"""
|
||||
def _retry_on_exception(e):
|
||||
return isinstance(e, exception)
|
||||
|
||||
def _backoff_sleep(previous_attempt_number, delay_since_first_attempt_ms):
|
||||
exp = backoff_rate ** previous_attempt_number
|
||||
wait_for = max(0, interval * exp)
|
||||
LOG.debug("Sleeping for %s seconds", wait_for)
|
||||
return wait_for * 1000.0
|
||||
|
||||
def _print_stop(previous_attempt_number, delay_since_first_attempt_ms):
|
||||
delay_since_first_attempt = delay_since_first_attempt_ms / 1000.0
|
||||
LOG.debug("Failed attempt %s", previous_attempt_number)
|
||||
LOG.debug("Have been at this for %s seconds",
|
||||
delay_since_first_attempt)
|
||||
return previous_attempt_number == retries
|
||||
|
||||
if retries < 1:
|
||||
raise ValueError(_('Retries must be greater than or '
|
||||
'equal to 1 (received: %s).') % retries)
|
||||
|
||||
def _decorator(f):
|
||||
|
||||
@six.wraps(f)
|
||||
def _wrapper(*args, **kwargs):
|
||||
r = retrying.Retrying(retry_on_exception=_retry_on_exception,
|
||||
wait_func=_backoff_sleep,
|
||||
stop_func=_print_stop)
|
||||
return r.call(f, *args, **kwargs)
|
||||
|
||||
return _wrapper
|
||||
|
||||
return _decorator
|
||||
|
Loading…
Reference in New Issue
Block a user