From 999a4d8bcb7b68b69627a30040ffb1ccbcba8f22 Mon Sep 17 00:00:00 2001
From: Rabi Mishra <ramishra@redhat.com>
Date: Thu, 12 Mar 2020 08:20:47 +0530
Subject: [PATCH] Retry transactions for DBConnectionError

Like DbDeadlock retry in DBConnectionError as oslo.db and SQLAlchemy
already automatically reconnect, if the database was disconnected
and a new transaction is beginning.

Change-Id: I479408fc47753e5971b347b379079b30e035cc08
---
 heat/db/sqlalchemy/api.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py
index a33b39d123..dd0505576a 100644
--- a/heat/db/sqlalchemy/api.py
+++ b/heat/db/sqlalchemy/api.py
@@ -90,23 +90,25 @@ def get_session():
     return get_facade().get_session()
 
 
-def retry_on_deadlock(func):
+def retry_on_db_error(func):
     @functools.wraps(func)
     def try_func(context, *args, **kwargs):
         if (context.session.transaction is None or
                 not context.session.autocommit):
             wrapped = oslo_db_api.wrap_db_retry(max_retries=3,
                                                 retry_on_deadlock=True,
+                                                retry_on_disconnect=True,
                                                 retry_interval=0.5,
                                                 inc_retry_interval=True)(func)
             return wrapped(context, *args, **kwargs)
         else:
             try:
                 return func(context, *args, **kwargs)
-            except db_exception.DBDeadlock:
+            except (db_exception.DBDeadlock, db_exception.DBConnectionError):
                 with excutils.save_and_reraise_exception():
-                    LOG.debug('Not retrying on DBDeadlock '
-                              'because transaction not closed')
+                    LOG.debug('Not retrying on DBDeadlock and '
+                              'DBConnectionError because '
+                              'transaction not closed')
     return try_func
 
 
@@ -265,7 +267,7 @@ def resource_get_all(context):
     return results
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def resource_purge_deleted(context, stack_id):
     filters = {'stack_id': stack_id, 'action': 'DELETE', 'status': 'COMPLETE'}
     query = context.session.query(models.Resource)
@@ -286,7 +288,7 @@ def _add_atomic_key_to_values(values, atomic_key):
         values['atomic_key'] = atomic_key + 1
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def resource_update(context, resource_id, values, atomic_key,
                     expected_engine_id=None):
     return _try_resource_update(context, resource_id, values, atomic_key,
@@ -483,7 +485,7 @@ def resource_create(context, values):
     return resource_ref
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def resource_create_replacement(context,
                                 existing_res_id, existing_res_values,
                                 new_res_values,
@@ -807,7 +809,7 @@ def stack_create(context, values):
     return stack_ref
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def stack_update(context, stack_id, values, exp_trvsl=None):
     session = context.session
     with session.begin(subtransactions=True):
@@ -865,7 +867,9 @@ def _is_duplicate_error(exc):
 
 
 @oslo_db_api.wrap_db_retry(max_retries=3, retry_on_deadlock=True,
-                           retry_interval=0.5, inc_retry_interval=True,
+                           retry_on_disconnect=True,
+                           retry_interval=0.5,
+                           inc_retry_interval=True,
                            exception_checker=_is_duplicate_error)
 def stack_lock_create(context, stack_id, engine_id):
     with db_context.writer.independent.using(context) as session:
@@ -1163,7 +1167,7 @@ def _delete_event_rows(context, stack_id, limit):
     return retval
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def event_create(context, values):
     if 'stack_id' in values and cfg.CONF.max_events_per_stack:
         # only count events and purge on average
@@ -1575,7 +1579,7 @@ def sync_point_delete_all_by_stack_and_traversal(context, stack_id,
     return rows_deleted
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def sync_point_create(context, values):
     values['entity_id'] = str(values['entity_id'])
     sync_point_ref = models.SyncPoint()
@@ -1591,7 +1595,7 @@ def sync_point_get(context, entity_id, traversal_id, is_update):
     )
 
 
-@retry_on_deadlock
+@retry_on_db_error
 def sync_point_update_input_data(context, entity_id,
                                  traversal_id, is_update, atomic_key,
                                  input_data):