diff --git a/heat/common/timeutils.py b/heat/common/timeutils.py index 2249cbac6..3fc273755 100644 --- a/heat/common/timeutils.py +++ b/heat/common/timeutils.py @@ -15,6 +15,7 @@ Utilities for handling ISO 8601 duration format. """ +import random import re @@ -39,3 +40,21 @@ def parse_isoduration(duration): t += int(result.group(3)) if result.group(3) else 0 return t + + +def retry_backoff_delay(attempt, scale_factor=1.0, jitter_max=0.0): + """ + Calculate an exponential backoff delay with jitter. + + Delay is calculated as + 2^attempt + (uniform random from [0,1) * jitter_max) + + :param attempt: The count of the current retry attempt + :param scale_factor: Multiplier to scale the exponential delay by + :param jitter_max: Maximum of random seconds to add to the delay + :returns: Seconds since epoch to wait until + """ + exp = float(2 ** attempt) * float(scale_factor) + if jitter_max == 0.0: + return exp + return exp + random.random() * jitter_max diff --git a/heat/tests/test_iso8601_utils.py b/heat/tests/test_timeutils.py similarity index 50% rename from heat/tests/test_iso8601_utils.py rename to heat/tests/test_timeutils.py index e3e0fbccf..66213d0ad 100644 --- a/heat/tests/test_iso8601_utils.py +++ b/heat/tests/test_timeutils.py @@ -11,6 +11,8 @@ # License for the specific language governing permissions and limitations # under the License. +from testtools import matchers + from heat.common import timeutils as util from heat.tests.common import HeatTestCase @@ -42,3 +44,101 @@ class ISO8601UtilityTest(HeatTestCase): self.assertRaises(ValueError, util.parse_isoduration, 'PT1MM') self.assertRaises(ValueError, util.parse_isoduration, 'PT1S0S') self.assertRaises(ValueError, util.parse_isoduration, 'ABCDEFGH') + + +class RetryBackoffExponentialTest(HeatTestCase): + + scenarios = [( + '0_0', + dict( + attempt=0, + scale_factor=0.0, + delay=0.0, + ) + ), ( + '0_1', + dict( + attempt=0, + scale_factor=1.0, + delay=1.0, + ) + ), ( + '1_1', + dict( + attempt=1, + scale_factor=1.0, + delay=2.0, + ) + ), ( + '2_1', + dict( + attempt=2, + scale_factor=1.0, + delay=4.0, + ) + ), ( + '3_1', + dict( + attempt=3, + scale_factor=1.0, + delay=8.0, + ) + ), ( + '4_1', + dict( + attempt=4, + scale_factor=1.0, + delay=16.0, + ) + ), ( + '4_4', + dict( + attempt=4, + scale_factor=4.0, + delay=64.0, + ) + )] + + def test_backoff_delay(self): + delay = util.retry_backoff_delay( + self.attempt, self.scale_factor) + self.assertEqual(delay, self.delay) + + +class RetryBackoffJitterTest(HeatTestCase): + + scenarios = [( + '0_0_1', + dict( + attempt=0, + scale_factor=0.0, + jitter_max=1.0, + delay_from=0.0, + delay_to=1.0 + ) + ), ( + '1_1_1', + dict( + attempt=1, + scale_factor=1.0, + jitter_max=1.0, + delay_from=2.0, + delay_to=3.0 + ) + ), ( + '1_1_5', + dict( + attempt=1, + scale_factor=1.0, + jitter_max=5.0, + delay_from=2.0, + delay_to=7.0 + ) + )] + + def test_backoff_delay(self): + for _ in range(100): + delay = util.retry_backoff_delay( + self.attempt, self.scale_factor, self.jitter_max) + self.assertThat(delay, matchers.GreaterThan(self.delay_from)) + self.assertThat(delay, matchers.LessThan(self.delay_to))