Merge "Retry on deadlock Transactions in backend"

keystone/identity/backends/sql.py

@@ -14,6 +14,7 @@
 
 import datetime
 
+from oslo_db import api as oslo_db_api
 import sqlalchemy
 
 from keystone.common import driver_hints
@@ -265,6 +266,7 @@ class Identity(base.IdentityDriverBase):
             query = sql.filter_limit_query(model.User, query, hints)
             return [base.filter_user(u.to_dict()) for u in query]
 
+    @oslo_db_api.wrap_db_retry(retry_on_deadlock=True)
     def delete_user(self, user_id):
         with sql.session_for_write() as session:
             ref = self._get_user(session, user_id)

keystone/tests/unit/test_v3_identity.py

@@ -18,6 +18,7 @@ import uuid
 import fixtures
 import freezegun
 import mock
+from oslo_db import exception as oslo_db_exception
 from oslo_log import log
 from six.moves import http_client
 from testtools import matchers
@@ -548,6 +549,43 @@ class IdentityTestCase(test_v3.RestfulTestCase):
         r = self.credential_api.get_credential(credential2['id'])
         self.assertDictEqual(credential2, r)
 
+    def test_delete_user_retries_on_deadlock(self):
+        patcher = mock.patch('sqlalchemy.orm.query.Query.delete',
+                             autospec=True)
+
+        class FakeDeadlock(object):
+            def __init__(self, mock_patcher):
+                self.deadlock_count = 2
+                self.mock_patcher = mock_patcher
+                self.patched = True
+
+            def __call__(self, *args, **kwargs):
+                if self.deadlock_count > 1:
+                    self.deadlock_count -= 1
+                else:
+                    self.mock_patcher.stop()
+                    self.patched = False
+                raise oslo_db_exception.DBDeadlock
+
+        sql_delete_mock = patcher.start()
+        side_effect = FakeDeadlock(patcher)
+        sql_delete_mock.side_effect = side_effect
+
+        user_ref = unit.create_user(self.identity_api,
+                                    domain_id=self.domain['id'])
+
+        try:
+            self.identity_api.delete_user(user_id=user_ref['id'])
+        finally:
+            if side_effect.patched:
+                patcher.stop()
+
+        call_count = sql_delete_mock.call_count
+
+        # initial attempt + 1 retry
+        delete_user_attempt_count = 2
+        self.assertEqual(call_count, delete_user_attempt_count)
+
     # group crud tests
 
     def test_create_group(self):