From 4a9d55a1b041007fc1b2340c5a143ffa2eb3f31d Mon Sep 17 00:00:00 2001
From: Renat Akhmerov <renat.akhmerov@gmail.com>
Date: Tue, 23 Jul 2019 16:59:57 +0700
Subject: [PATCH] Retry a DB transaction on "Too many connections" error

* Writing a unit test is very problematic but the fix has been
  tested manually.

Closes-Bug: #1837532

Change-Id: I4fa15994a7359a5f90a0a4671d47b19fe928cf33
---
 lower-constraints.txt |  2 +-
 mistral/db/utils.py   | 19 ++++++++++++++-----
 requirements.txt      |  2 +-
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/lower-constraints.txt b/lower-constraints.txt
index acf70c44e..8e5ed1a14 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -159,7 +159,7 @@ stestr==2.0.0
 stevedore==1.20.0
 tempest==17.1.0
 Tempita==0.5.2
-tenacity==4.4.0
+tenacity==5.0.1
 testrepository==0.0.18
 testtools==2.2.0
 tooz==1.58.0
diff --git a/mistral/db/utils.py b/mistral/db/utils.py
index e5ddba5b3..ca8750842 100644
--- a/mistral/db/utils.py
+++ b/mistral/db/utils.py
@@ -52,10 +52,14 @@ def _with_auth_context(auth_ctx, func, *args, **kw):
 
     try:
         return func(*args, **kw)
-    except _RETRY_ERRORS:
-        LOG.exception(
-            "DB error detected, operation will be retried: %s", func
-        )
+    except Exception as e:
+        # Note (rakhmerov): In case of "Too many connections" error from the
+        # database it doesn't get wrapped with a SQLAlchemy exception for some
+        # reason so we have to check the exception message explicitly.
+        if isinstance(e, _RETRY_ERRORS) or 'Too many connections' in str(e):
+            LOG.exception(
+                "DB error detected, operation will be retried: %s", func
+            )
 
         raise
     finally:
@@ -75,7 +79,12 @@ def retry_on_db_error(func, retry=None):
     """
     if not retry:
         retry = tenacity.Retrying(
-            retry=tenacity.retry_if_exception_type(_RETRY_ERRORS),
+            retry=(
+                tenacity.retry_if_exception_type(_RETRY_ERRORS) |
+                tenacity.retry_if_exception_message(
+                    match='Too many connections'
+                )
+            ),
             stop=tenacity.stop_after_attempt(50),
             wait=tenacity.wait_incrementing(start=0, increment=0.1, max=2)
         )
diff --git a/requirements.txt b/requirements.txt
index 6eb5d33f3..1e1accf4e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -58,7 +58,7 @@ python-qinlingclient>=1.0.0 # Apache-2.0
 PyJWT>=1.0.1 # MIT
 PyYAML>=3.12 # MIT
 requests>=2.14.2 # Apache-2.0
-tenacity>=4.4.0 # Apache-2.0
+tenacity>=5.0.1 # Apache-2.0
 six>=1.10.0 # MIT
 SQLAlchemy>=1.2.5 # MIT
 stevedore>=1.20.0 # Apache-2.0