timeutils function retry_backoff_delay

Calculates an exponential backoff delay with jitter for retrying
operations which might fail.

The exponential delay on each attempt gives the service a chance
to process its backlog of retries after it starts responding again.

The jitter adds some randomness to the delay to prevent
multiple synchronised backoff retries from hitting the
service at exactly the same time.

The next change in this series makes use of this function.

Change-Id: Ie209ef62c0da68a5b503354764e5add2b6e5569a

heat/common/timeutils.py

@@ -15,6 +15,7 @@
 Utilities for handling ISO 8601 duration format.
 """
 
+import random
 import re
 
 
@@ -39,3 +40,21 @@ def parse_isoduration(duration):
     t += int(result.group(3)) if result.group(3) else 0
 
     return t
+
+
+def retry_backoff_delay(attempt, scale_factor=1.0, jitter_max=0.0):
+    """
+    Calculate an exponential backoff delay with jitter.
+
+    Delay is calculated as
+    2^attempt + (uniform random from [0,1) * jitter_max)
+
+    :param attempt: The count of the current retry attempt
+    :param scale_factor: Multiplier to scale the exponential delay by
+    :param jitter_max: Maximum of random seconds to add to the delay
+    :returns: Seconds since epoch to wait until
+    """
+    exp = float(2 ** attempt) * float(scale_factor)
+    if jitter_max == 0.0:
+        return exp
+    return exp + random.random() * jitter_max

heat/tests/test_timeutils.py

@@ -11,6 +11,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from testtools import matchers
+
 from heat.common import timeutils as util
 from heat.tests.common import HeatTestCase
 
@@ -42,3 +44,101 @@ class ISO8601UtilityTest(HeatTestCase):
         self.assertRaises(ValueError, util.parse_isoduration, 'PT1MM')
         self.assertRaises(ValueError, util.parse_isoduration, 'PT1S0S')
         self.assertRaises(ValueError, util.parse_isoduration, 'ABCDEFGH')
+
+
+class RetryBackoffExponentialTest(HeatTestCase):
+
+    scenarios = [(
+        '0_0',
+        dict(
+            attempt=0,
+            scale_factor=0.0,
+            delay=0.0,
+        )
+    ), (
+        '0_1',
+        dict(
+            attempt=0,
+            scale_factor=1.0,
+            delay=1.0,
+        )
+    ), (
+        '1_1',
+        dict(
+            attempt=1,
+            scale_factor=1.0,
+            delay=2.0,
+        )
+    ), (
+        '2_1',
+        dict(
+            attempt=2,
+            scale_factor=1.0,
+            delay=4.0,
+        )
+    ), (
+        '3_1',
+        dict(
+            attempt=3,
+            scale_factor=1.0,
+            delay=8.0,
+        )
+    ), (
+        '4_1',
+        dict(
+            attempt=4,
+            scale_factor=1.0,
+            delay=16.0,
+        )
+    ), (
+        '4_4',
+        dict(
+            attempt=4,
+            scale_factor=4.0,
+            delay=64.0,
+        )
+    )]
+
+    def test_backoff_delay(self):
+        delay = util.retry_backoff_delay(
+            self.attempt, self.scale_factor)
+        self.assertEqual(delay, self.delay)
+
+
+class RetryBackoffJitterTest(HeatTestCase):
+
+    scenarios = [(
+        '0_0_1',
+        dict(
+            attempt=0,
+            scale_factor=0.0,
+            jitter_max=1.0,
+            delay_from=0.0,
+            delay_to=1.0
+        )
+    ), (
+        '1_1_1',
+        dict(
+            attempt=1,
+            scale_factor=1.0,
+            jitter_max=1.0,
+            delay_from=2.0,
+            delay_to=3.0
+        )
+    ), (
+        '1_1_5',
+        dict(
+            attempt=1,
+            scale_factor=1.0,
+            jitter_max=5.0,
+            delay_from=2.0,
+            delay_to=7.0
+        )
+    )]
+
+    def test_backoff_delay(self):
+        for _ in range(100):
+            delay = util.retry_backoff_delay(
+                self.attempt, self.scale_factor, self.jitter_max)
+            self.assertThat(delay, matchers.GreaterThan(self.delay_from))
+            self.assertThat(delay, matchers.LessThan(self.delay_to))