Use oslo.db jitter in our db retries

In theory, two parallel threads that need backoff, with the same
backoff timers, can synchronize and just repeatedly hose themselves.
oslo.db already has a way to put some randomness into the retry
logic, so let's try that out.

Change-Id: I2090eb1ca5f3c09500069309d147880081b8213e

neutron_lib/db/api.py

@@ -12,6 +12,7 @@
 
 import contextlib
 import copy
+import inspect
 import weakref
 
 from oslo_concurrency import lockutils
@@ -147,11 +148,24 @@ def _copy_if_lds(item):
     return copy.deepcopy(item) if isinstance(item, (list, dict, set)) else item
 
 
+# Do retries with some randomness in the backoff, if the version
+# of oslo.db supports it.
+# TODO(dougwig): remove this gross if/else when lower constraints
+# on oslo.db is >= 4.37.0.
+if 'jitter' in inspect.getargspec(oslo_db_api.wrap_db_retry.__init__).args:
     _retry_db_errors = oslo_db_api.wrap_db_retry(
         max_retries=MAX_RETRIES,
         retry_interval=0.5,
         inc_retry_interval=True,
-    exception_checker=is_retriable
+        exception_checker=is_retriable,
+        jitter=True
+    )
+else:
+    _retry_db_errors = oslo_db_api.wrap_db_retry(
+        max_retries=MAX_RETRIES,
+        retry_interval=0.5,
+        inc_retry_interval=True,
+        exception_checker=is_retriable,
     )
 
 

neutron_lib/tests/unit/db/test_api.py

@@ -158,7 +158,7 @@ class TestDeadLockDecorator(_base.BaseTestCase):
         if exc_to_raise == db_exc.DBDeadlock:
             self.assertEqual(True, (fake_timer.counter <= sum(worst_case)))
         else:
-            self.assertEqual(sum(worst_case), fake_timer.counter)
+            self.assertTrue(sum(worst_case) >= fake_timer.counter)
 
     def test_all_deadlock_time_elapsed(self):
         self._test_retry_time_cost(db_exc.DBDeadlock)

releasenotes/notes/oslo-db-jitter-c4d13cc81755203e.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - The oslo.db wrap_db_retry function now supports randomized time jitter
+    in its retry algorithm. Add support for that feature, if the installed
+    version of oslo.db supports it.