From 6a8f50bb914334c6cf2f5aacb6bcb232e3897570 Mon Sep 17 00:00:00 2001
From: Steve Baker <sbaker@redhat.com>
Date: Mon, 21 Jul 2014 15:14:34 -0400
Subject: [PATCH] timeutils function retry_backoff_delay

Calculates an exponential backoff delay with jitter for retrying
operations which might fail.

The exponential delay on each attempt gives the service a chance
to process its backlog of retries after it starts responding again.

The jitter adds some randomness to the delay to prevent
multiple synchronised backoff retries from hitting the
service at exactly the same time.

The next change in this series makes use of this function.

Change-Id: Ie209ef62c0da68a5b503354764e5add2b6e5569a
---
 heat/common/timeutils.py                      |  19 ++++
 ...est_iso8601_utils.py => test_timeutils.py} | 100 ++++++++++++++++++
 2 files changed, 119 insertions(+)
 rename heat/tests/{test_iso8601_utils.py => test_timeutils.py} (50%)

diff --git a/heat/common/timeutils.py b/heat/common/timeutils.py
index 2249cbac6..3fc273755 100644
--- a/heat/common/timeutils.py
+++ b/heat/common/timeutils.py
@@ -15,6 +15,7 @@
 Utilities for handling ISO 8601 duration format.
 """
 
+import random
 import re
 
 
@@ -39,3 +40,21 @@ def parse_isoduration(duration):
     t += int(result.group(3)) if result.group(3) else 0
 
     return t
+
+
+def retry_backoff_delay(attempt, scale_factor=1.0, jitter_max=0.0):
+    """
+    Calculate an exponential backoff delay with jitter.
+
+    Delay is calculated as
+    2^attempt + (uniform random from [0,1) * jitter_max)
+
+    :param attempt: The count of the current retry attempt
+    :param scale_factor: Multiplier to scale the exponential delay by
+    :param jitter_max: Maximum of random seconds to add to the delay
+    :returns: Seconds since epoch to wait until
+    """
+    exp = float(2 ** attempt) * float(scale_factor)
+    if jitter_max == 0.0:
+        return exp
+    return exp + random.random() * jitter_max
diff --git a/heat/tests/test_iso8601_utils.py b/heat/tests/test_timeutils.py
similarity index 50%
rename from heat/tests/test_iso8601_utils.py
rename to heat/tests/test_timeutils.py
index e3e0fbccf..66213d0ad 100644
--- a/heat/tests/test_iso8601_utils.py
+++ b/heat/tests/test_timeutils.py
@@ -11,6 +11,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from testtools import matchers
+
 from heat.common import timeutils as util
 from heat.tests.common import HeatTestCase
 
@@ -42,3 +44,101 @@ class ISO8601UtilityTest(HeatTestCase):
         self.assertRaises(ValueError, util.parse_isoduration, 'PT1MM')
         self.assertRaises(ValueError, util.parse_isoduration, 'PT1S0S')
         self.assertRaises(ValueError, util.parse_isoduration, 'ABCDEFGH')
+
+
+class RetryBackoffExponentialTest(HeatTestCase):
+
+    scenarios = [(
+        '0_0',
+        dict(
+            attempt=0,
+            scale_factor=0.0,
+            delay=0.0,
+        )
+    ), (
+        '0_1',
+        dict(
+            attempt=0,
+            scale_factor=1.0,
+            delay=1.0,
+        )
+    ), (
+        '1_1',
+        dict(
+            attempt=1,
+            scale_factor=1.0,
+            delay=2.0,
+        )
+    ), (
+        '2_1',
+        dict(
+            attempt=2,
+            scale_factor=1.0,
+            delay=4.0,
+        )
+    ), (
+        '3_1',
+        dict(
+            attempt=3,
+            scale_factor=1.0,
+            delay=8.0,
+        )
+    ), (
+        '4_1',
+        dict(
+            attempt=4,
+            scale_factor=1.0,
+            delay=16.0,
+        )
+    ), (
+        '4_4',
+        dict(
+            attempt=4,
+            scale_factor=4.0,
+            delay=64.0,
+        )
+    )]
+
+    def test_backoff_delay(self):
+        delay = util.retry_backoff_delay(
+            self.attempt, self.scale_factor)
+        self.assertEqual(delay, self.delay)
+
+
+class RetryBackoffJitterTest(HeatTestCase):
+
+    scenarios = [(
+        '0_0_1',
+        dict(
+            attempt=0,
+            scale_factor=0.0,
+            jitter_max=1.0,
+            delay_from=0.0,
+            delay_to=1.0
+        )
+    ), (
+        '1_1_1',
+        dict(
+            attempt=1,
+            scale_factor=1.0,
+            jitter_max=1.0,
+            delay_from=2.0,
+            delay_to=3.0
+        )
+    ), (
+        '1_1_5',
+        dict(
+            attempt=1,
+            scale_factor=1.0,
+            jitter_max=5.0,
+            delay_from=2.0,
+            delay_to=7.0
+        )
+    )]
+
+    def test_backoff_delay(self):
+        for _ in range(100):
+            delay = util.retry_backoff_delay(
+                self.attempt, self.scale_factor, self.jitter_max)
+            self.assertThat(delay, matchers.GreaterThan(self.delay_from))
+            self.assertThat(delay, matchers.LessThan(self.delay_to))