From fada2b8d725376750dba2d3f0818daf8efb8d80e Mon Sep 17 00:00:00 2001
From: Doug Wiegley <doug@parkside.io>
Date: Thu, 17 Jan 2019 19:39:06 -0700
Subject: [PATCH] Use oslo.db jitter in our db retries

In theory, two parallel threads that need backoff, with the same
backoff timers, can synchronize and just repeatedly hose themselves.
oslo.db already has a way to put some randomness into the retry
logic, so let's try that out.

Change-Id: I2090eb1ca5f3c09500069309d147880081b8213e
---
 neutron_lib/db/api.py                         | 26 ++++++++++++++-----
 neutron_lib/tests/unit/db/test_api.py         |  2 +-
 .../oslo-db-jitter-c4d13cc81755203e.yaml      |  5 ++++
 3 files changed, 26 insertions(+), 7 deletions(-)
 create mode 100644 releasenotes/notes/oslo-db-jitter-c4d13cc81755203e.yaml

diff --git a/neutron_lib/db/api.py b/neutron_lib/db/api.py
index 874f0f822..9374b12ab 100644
--- a/neutron_lib/db/api.py
+++ b/neutron_lib/db/api.py
@@ -12,6 +12,7 @@
 
 import contextlib
 import copy
+import inspect
 import weakref
 
 from oslo_concurrency import lockutils
@@ -147,12 +148,25 @@ def _copy_if_lds(item):
     return copy.deepcopy(item) if isinstance(item, (list, dict, set)) else item
 
 
-_retry_db_errors = oslo_db_api.wrap_db_retry(
-    max_retries=MAX_RETRIES,
-    retry_interval=0.5,
-    inc_retry_interval=True,
-    exception_checker=is_retriable
-)
+# Do retries with some randomness in the backoff, if the version
+# of oslo.db supports it.
+# TODO(dougwig): remove this gross if/else when lower constraints
+# on oslo.db is >= 4.37.0.
+if 'jitter' in inspect.getargspec(oslo_db_api.wrap_db_retry.__init__).args:
+    _retry_db_errors = oslo_db_api.wrap_db_retry(
+        max_retries=MAX_RETRIES,
+        retry_interval=0.5,
+        inc_retry_interval=True,
+        exception_checker=is_retriable,
+        jitter=True
+    )
+else:
+    _retry_db_errors = oslo_db_api.wrap_db_retry(
+        max_retries=MAX_RETRIES,
+        retry_interval=0.5,
+        inc_retry_interval=True,
+        exception_checker=is_retriable,
+    )
 
 
 def retry_db_errors(f):
diff --git a/neutron_lib/tests/unit/db/test_api.py b/neutron_lib/tests/unit/db/test_api.py
index c12969569..32b347c16 100644
--- a/neutron_lib/tests/unit/db/test_api.py
+++ b/neutron_lib/tests/unit/db/test_api.py
@@ -158,7 +158,7 @@ class TestDeadLockDecorator(_base.BaseTestCase):
         if exc_to_raise == db_exc.DBDeadlock:
             self.assertEqual(True, (fake_timer.counter <= sum(worst_case)))
         else:
-            self.assertEqual(sum(worst_case), fake_timer.counter)
+            self.assertTrue(sum(worst_case) >= fake_timer.counter)
 
     def test_all_deadlock_time_elapsed(self):
         self._test_retry_time_cost(db_exc.DBDeadlock)
diff --git a/releasenotes/notes/oslo-db-jitter-c4d13cc81755203e.yaml b/releasenotes/notes/oslo-db-jitter-c4d13cc81755203e.yaml
new file mode 100644
index 000000000..db583b740
--- /dev/null
+++ b/releasenotes/notes/oslo-db-jitter-c4d13cc81755203e.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - The oslo.db wrap_db_retry function now supports randomized time jitter
+    in its retry algorithm. Add support for that feature, if the installed
+    version of oslo.db supports it.