Retry a DB transaction on "Too many connections" error
* Writing a unit test is very problematic but the fix has been tested manually. Closes-Bug: #1837532 Change-Id: I4fa15994a7359a5f90a0a4671d47b19fe928cf33
This commit is contained in:
parent
f2e8d614d9
commit
4a9d55a1b0
|
@ -159,7 +159,7 @@ stestr==2.0.0
|
|||
stevedore==1.20.0
|
||||
tempest==17.1.0
|
||||
Tempita==0.5.2
|
||||
tenacity==4.4.0
|
||||
tenacity==5.0.1
|
||||
testrepository==0.0.18
|
||||
testtools==2.2.0
|
||||
tooz==1.58.0
|
||||
|
|
|
@ -52,10 +52,14 @@ def _with_auth_context(auth_ctx, func, *args, **kw):
|
|||
|
||||
try:
|
||||
return func(*args, **kw)
|
||||
except _RETRY_ERRORS:
|
||||
LOG.exception(
|
||||
"DB error detected, operation will be retried: %s", func
|
||||
)
|
||||
except Exception as e:
|
||||
# Note (rakhmerov): In case of "Too many connections" error from the
|
||||
# database it doesn't get wrapped with a SQLAlchemy exception for some
|
||||
# reason so we have to check the exception message explicitly.
|
||||
if isinstance(e, _RETRY_ERRORS) or 'Too many connections' in str(e):
|
||||
LOG.exception(
|
||||
"DB error detected, operation will be retried: %s", func
|
||||
)
|
||||
|
||||
raise
|
||||
finally:
|
||||
|
@ -75,7 +79,12 @@ def retry_on_db_error(func, retry=None):
|
|||
"""
|
||||
if not retry:
|
||||
retry = tenacity.Retrying(
|
||||
retry=tenacity.retry_if_exception_type(_RETRY_ERRORS),
|
||||
retry=(
|
||||
tenacity.retry_if_exception_type(_RETRY_ERRORS) |
|
||||
tenacity.retry_if_exception_message(
|
||||
match='Too many connections'
|
||||
)
|
||||
),
|
||||
stop=tenacity.stop_after_attempt(50),
|
||||
wait=tenacity.wait_incrementing(start=0, increment=0.1, max=2)
|
||||
)
|
||||
|
|
|
@ -58,7 +58,7 @@ python-qinlingclient>=1.0.0 # Apache-2.0
|
|||
PyJWT>=1.0.1 # MIT
|
||||
PyYAML>=3.12 # MIT
|
||||
requests>=2.14.2 # Apache-2.0
|
||||
tenacity>=4.4.0 # Apache-2.0
|
||||
tenacity>=5.0.1 # Apache-2.0
|
||||
six>=1.10.0 # MIT
|
||||
SQLAlchemy>=1.2.5 # MIT
|
||||
stevedore>=1.20.0 # Apache-2.0
|
||||
|
|
Loading…
Reference in New Issue