Merge "Add a _retry_on_deadlock decorator"

glance/db/sqlalchemy/api.py

@@ -21,6 +21,7 @@
 """Defines interface for DB access."""
 
 from oslo.config import cfg
+from retrying import retry
 import six
 from six.moves import xrange
 import sqlalchemy
@@ -52,6 +53,15 @@ CONF.import_opt('connection', 'glance.openstack.common.db.options',
 _FACADE = None
 
 
+def _retry_on_deadlock(exc):
+    """Decorator to retry a DB API call if Deadlock was received."""
+
+    if isinstance(exc, db_exception.DBDeadlock):
+        LOG.warn(_("Deadlock detected. Retrying..."))
+        return True
+    return False
+
+
 def _create_facade_lazily():
     global _FACADE
     if _FACADE is None:
@@ -632,6 +642,8 @@ def _update_values(image_ref, values):
             setattr(image_ref, k, values[k])
 
 
+@retry(retry_on_exception=_retry_on_deadlock, wait_fixed=500,
+       stop_max_attempt_number=50)
 def _image_update(context, values, image_id, purge_props=False,
                   from_state=None):
     """

glance/tests/unit/test_db.py

@@ -24,6 +24,8 @@ from glance.common import exception
 from glance.common import utils
 import glance.context
 import glance.db
+from glance.db.sqlalchemy import api
+from glance.openstack.common.db import exception as db_exc
 import glance.tests.unit.utils as unit_test_utils
 import glance.tests.utils as test_utils
 
@@ -689,3 +691,29 @@ class TestTaskRepo(test_utils.BaseTestCase):
         self.assertRaises(exception.NotFound,
                           self.task_repo.get,
                           task.task_id)
+
+
+class RetryOnDeadlockTestCase(test_utils.BaseTestCase):
+
+    def test_raise_deadlock(self):
+
+        class TestException(Exception):
+            pass
+
+        self.attempts = 3
+
+        def _mock_get_session():
+            def _raise_exceptions():
+                self.attempts -= 1
+                if self.attempts <= 0:
+                    raise TestException("Exit")
+                raise db_exc.DBDeadlock("Fake Exception")
+            return _raise_exceptions
+
+        with mock.patch.object(api, 'get_session') as sess:
+            sess.side_effect = _mock_get_session()
+
+            try:
+                api._image_update(None, {}, 'fake-id')
+            except TestException:
+                self.assertEqual(sess.call_count, 3)