Enable retries in redis driver
this is followup to Iaab5ce609c0dcf7085f5dd43efbd37eb4b88f17b actually retry for specified number of retries instead of raising error on first ConnectionError Change-Id: Ibca3f568b65dfea252da4b67f6d5105ba7f1ecb1 (cherry picked from commit47c4d56e44
) (cherry picked from commit11526e594c
) (cherry picked from commitc1ae1649d2
)
This commit is contained in:
parent
667e2ec1f4
commit
b7a5b073f8
|
@ -0,0 +1,5 @@
|
|||
---
|
||||
features:
|
||||
- |
|
||||
Redis driver retries actions for up to 15 times when met with error
|
||||
connecting to Redis.
|
|
@ -40,6 +40,7 @@ from tooz import utils
|
|||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
@contextlib.contextmanager
|
||||
def _translate_failures():
|
||||
"""Translates common redis exceptions into tooz exceptions."""
|
||||
|
@ -53,6 +54,48 @@ def _translate_failures():
|
|||
utils.raise_with_cause(tooz.ToozError,
|
||||
encodeutils.exception_to_unicode(e),
|
||||
cause=e)
|
||||
=======
|
||||
def _handle_failures(n_tries=15):
|
||||
|
||||
"""Translates common redis exceptions into tooz exceptions.
|
||||
|
||||
This also enables retrying on certain exceptions.
|
||||
|
||||
:param func: the function to act on
|
||||
:param n_tries: the number of retries
|
||||
"""
|
||||
def inner(func):
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
ntries = n_tries
|
||||
while ntries:
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except exceptions.ConnectionError as e:
|
||||
# retry ntries times and then raise a connection error
|
||||
ntries -= 1
|
||||
if not ntries:
|
||||
LOG.debug(
|
||||
"Redis connection error, "
|
||||
"retry limit has been reached, aborting - %s", e
|
||||
)
|
||||
utils.raise_with_cause(
|
||||
coordination.ToozConnectionError,
|
||||
encodeutils.exception_to_unicode(e),
|
||||
cause=e)
|
||||
LOG.debug("Redis connection error, will retry - %s", e)
|
||||
|
||||
except (exceptions.TimeoutError) as e:
|
||||
utils.raise_with_cause(coordination.ToozConnectionError,
|
||||
encodeutils.exception_to_unicode(e),
|
||||
cause=e)
|
||||
except exceptions.RedisError as e:
|
||||
utils.raise_with_cause(tooz.ToozError,
|
||||
encodeutils.exception_to_unicode(e),
|
||||
cause=e)
|
||||
return wrapper
|
||||
return inner
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
|
||||
|
||||
class RedisLock(locking.Lock):
|
||||
|
@ -68,6 +111,10 @@ class RedisLock(locking.Lock):
|
|||
self._coord = coord
|
||||
self._client = client
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def is_still_owner(self):
|
||||
with _translate_failures():
|
||||
lock_tok = self._lock.local.token
|
||||
|
@ -76,10 +123,18 @@ class RedisLock(locking.Lock):
|
|||
owner_tok = self._client.get(self.name)
|
||||
return owner_tok == lock_tok
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def break_(self):
|
||||
with _translate_failures():
|
||||
return bool(self._client.delete(self.name))
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def acquire(self, blocking=True, shared=False):
|
||||
if shared:
|
||||
raise tooz.NotImplemented
|
||||
|
@ -92,6 +147,10 @@ class RedisLock(locking.Lock):
|
|||
self._coord._acquired_locks.add(self)
|
||||
return acquired
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def release(self):
|
||||
with self._exclusive_access:
|
||||
with _translate_failures():
|
||||
|
@ -104,6 +163,10 @@ class RedisLock(locking.Lock):
|
|||
self._coord._acquired_locks.discard(self)
|
||||
return True
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def heartbeat(self):
|
||||
with self._exclusive_access:
|
||||
if self.acquired:
|
||||
|
@ -438,6 +501,10 @@ return 1
|
|||
return master_client
|
||||
return redis.StrictRedis(**kwargs)
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def _start(self):
|
||||
super(RedisDriver, self)._start()
|
||||
try:
|
||||
|
@ -511,6 +578,10 @@ return 1
|
|||
def _decode_group_id(self, group_id):
|
||||
return utils.to_binary(group_id, encoding=self._encoding)
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def heartbeat(self):
|
||||
with _translate_failures():
|
||||
beat_id = self._encode_beat_id(self._member_id)
|
||||
|
@ -525,6 +596,10 @@ return 1
|
|||
exc_info=True)
|
||||
return min(self.lock_timeout, self.membership_timeout)
|
||||
|
||||
<<<<<<< HEAD (667e2e Fix broken CI and PyMySQL support)
|
||||
=======
|
||||
@_handle_failures()
|
||||
>>>>>>> CHANGE (c1ae16 Enable retries in redis driver)
|
||||
def _stop(self):
|
||||
while self._acquired_locks:
|
||||
lock = self._acquired_locks.pop()
|
||||
|
|
Loading…
Reference in New Issue