Retry a DB transaction on "Too many connections" error

* Writing a unit test is very problematic but the fix has been
  tested manually.

Closes-Bug: #1837532

Change-Id: I4fa15994a7359a5f90a0a4671d47b19fe928cf33

lower-constraints.txt

@@ -159,7 +159,7 @@ stestr==2.0.0
 stevedore==1.20.0
 tempest==17.1.0
 Tempita==0.5.2
-tenacity==4.4.0
+tenacity==5.0.1
 testrepository==0.0.18
 testtools==2.2.0
 tooz==1.58.0

mistral/db/utils.py

@@ -52,10 +52,14 @@ def _with_auth_context(auth_ctx, func, *args, **kw):
 
     try:
         return func(*args, **kw)
-    except _RETRY_ERRORS:
-        LOG.exception(
-            "DB error detected, operation will be retried: %s", func
-        )
+    except Exception as e:
+        # Note (rakhmerov): In case of "Too many connections" error from the
+        # database it doesn't get wrapped with a SQLAlchemy exception for some
+        # reason so we have to check the exception message explicitly.
+        if isinstance(e, _RETRY_ERRORS) or 'Too many connections' in str(e):
+            LOG.exception(
+                "DB error detected, operation will be retried: %s", func
+            )
 
         raise
     finally:
@@ -75,7 +79,12 @@ def retry_on_db_error(func, retry=None):
     """
     if not retry:
         retry = tenacity.Retrying(
-            retry=tenacity.retry_if_exception_type(_RETRY_ERRORS),
+            retry=(
+                tenacity.retry_if_exception_type(_RETRY_ERRORS) |
+                tenacity.retry_if_exception_message(
+                    match='Too many connections'
+                )
+            ),
             stop=tenacity.stop_after_attempt(50),
             wait=tenacity.wait_incrementing(start=0, increment=0.1, max=2)
         )

requirements.txt

@@ -58,7 +58,7 @@ python-qinlingclient>=1.0.0 # Apache-2.0
 PyJWT>=1.0.1 # MIT
 PyYAML>=3.12 # MIT
 requests>=2.14.2 # Apache-2.0
-tenacity>=4.4.0 # Apache-2.0
+tenacity>=5.0.1 # Apache-2.0
 six>=1.10.0 # MIT
 SQLAlchemy>=1.2.5 # MIT
 stevedore>=1.20.0 # Apache-2.0