Merge "Refactor timeout calculation to utility"
This commit is contained in:
commit
0ec488ee54
|
@ -17,11 +17,31 @@ Utilities for handling ISO 8601 duration format.
|
||||||
|
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
|
import time
|
||||||
|
|
||||||
from heat.common.i18n import _
|
from heat.common.i18n import _
|
||||||
|
|
||||||
|
|
||||||
iso_duration_re = re.compile('PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$')
|
iso_duration_re = re.compile('PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?$')
|
||||||
|
wallclock = time.time
|
||||||
|
|
||||||
|
|
||||||
|
class Duration(object):
|
||||||
|
'''
|
||||||
|
Note that we don't attempt to handle leap seconds or large clock
|
||||||
|
jumps here. The latter are assumed to be rare and the former
|
||||||
|
negligible in the context of the timeout. Time zone adjustments,
|
||||||
|
Daylight Savings and the like *are* handled. PEP 418 adds a proper
|
||||||
|
monotonic clock, but only in Python 3.3.
|
||||||
|
'''
|
||||||
|
def __init__(self, timeout=0):
|
||||||
|
self._endtime = wallclock() + timeout
|
||||||
|
|
||||||
|
def expired(self):
|
||||||
|
return wallclock() > self._endtime
|
||||||
|
|
||||||
|
def endtime(self):
|
||||||
|
return self._endtime
|
||||||
|
|
||||||
|
|
||||||
def parse_isoduration(duration):
|
def parse_isoduration(duration):
|
||||||
|
|
|
@ -14,7 +14,6 @@
|
||||||
import functools
|
import functools
|
||||||
import itertools
|
import itertools
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
import types
|
import types
|
||||||
|
|
||||||
import eventlet
|
import eventlet
|
||||||
|
@ -26,13 +25,13 @@ from six import reraise as raise_
|
||||||
|
|
||||||
from heat.common.i18n import _
|
from heat.common.i18n import _
|
||||||
from heat.common.i18n import _LI
|
from heat.common.i18n import _LI
|
||||||
|
from heat.common import timeutils
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
# Whether TaskRunner._sleep actually does an eventlet sleep when called.
|
# Whether TaskRunner._sleep actually does an eventlet sleep when called.
|
||||||
ENABLE_SLEEP = True
|
ENABLE_SLEEP = True
|
||||||
wallclock = time.time
|
|
||||||
|
|
||||||
|
|
||||||
def task_description(task):
|
def task_description(task):
|
||||||
|
@ -68,15 +67,10 @@ class Timeout(BaseException):
|
||||||
message = _('%s Timed out') % six.text_type(task_runner)
|
message = _('%s Timed out') % six.text_type(task_runner)
|
||||||
super(Timeout, self).__init__(message)
|
super(Timeout, self).__init__(message)
|
||||||
|
|
||||||
# Note that we don't attempt to handle leap seconds or large clock
|
self._duration = timeutils.Duration(timeout)
|
||||||
# jumps here. The latter are assumed to be rare and the former
|
|
||||||
# negligible in the context of the timeout. Time zone adjustments,
|
|
||||||
# Daylight Savings and the like *are* handled. PEP 418 adds a proper
|
|
||||||
# monotonic clock, but only in Python 3.3.
|
|
||||||
self._endtime = wallclock() + timeout
|
|
||||||
|
|
||||||
def expired(self):
|
def expired(self):
|
||||||
return wallclock() > self._endtime
|
return self._duration.expired()
|
||||||
|
|
||||||
def trigger(self, generator):
|
def trigger(self, generator):
|
||||||
"""Trigger the timeout on a given generator."""
|
"""Trigger the timeout on a given generator."""
|
||||||
|
@ -107,7 +101,7 @@ class Timeout(BaseException):
|
||||||
def __lt__(self, other):
|
def __lt__(self, other):
|
||||||
if not isinstance(other, Timeout):
|
if not isinstance(other, Timeout):
|
||||||
return NotImplemented
|
return NotImplemented
|
||||||
return self._endtime < other._endtime
|
return self._duration.endtime() < other._duration.endtime()
|
||||||
|
|
||||||
def __cmp__(self, other):
|
def __cmp__(self, other):
|
||||||
return self < other
|
return self < other
|
||||||
|
|
|
@ -15,6 +15,7 @@ import contextlib
|
||||||
|
|
||||||
import eventlet
|
import eventlet
|
||||||
|
|
||||||
|
from heat.common import timeutils
|
||||||
from heat.engine import dependencies
|
from heat.engine import dependencies
|
||||||
from heat.engine import scheduler
|
from heat.engine import scheduler
|
||||||
from heat.tests import common
|
from heat.tests import common
|
||||||
|
@ -647,16 +648,16 @@ class TaskTest(common.HeatTestCase):
|
||||||
self.assertTrue(runner.step())
|
self.assertTrue(runner.step())
|
||||||
|
|
||||||
def test_timeout(self):
|
def test_timeout(self):
|
||||||
st = scheduler.wallclock()
|
st = timeutils.wallclock()
|
||||||
|
|
||||||
def task():
|
def task():
|
||||||
while True:
|
while True:
|
||||||
yield
|
yield
|
||||||
|
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
scheduler.wallclock().AndReturn(st)
|
timeutils.wallclock().AndReturn(st)
|
||||||
scheduler.wallclock().AndReturn(st + 0.5)
|
timeutils.wallclock().AndReturn(st + 0.5)
|
||||||
scheduler.wallclock().AndReturn(st + 1.5)
|
timeutils.wallclock().AndReturn(st + 1.5)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
@ -667,7 +668,7 @@ class TaskTest(common.HeatTestCase):
|
||||||
self.assertRaises(scheduler.Timeout, runner.step)
|
self.assertRaises(scheduler.Timeout, runner.step)
|
||||||
|
|
||||||
def test_timeout_return(self):
|
def test_timeout_return(self):
|
||||||
st = scheduler.wallclock()
|
st = timeutils.wallclock()
|
||||||
|
|
||||||
def task():
|
def task():
|
||||||
while True:
|
while True:
|
||||||
|
@ -676,10 +677,10 @@ class TaskTest(common.HeatTestCase):
|
||||||
except scheduler.Timeout:
|
except scheduler.Timeout:
|
||||||
return
|
return
|
||||||
|
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
scheduler.wallclock().AndReturn(st)
|
timeutils.wallclock().AndReturn(st)
|
||||||
scheduler.wallclock().AndReturn(st + 0.5)
|
timeutils.wallclock().AndReturn(st + 0.5)
|
||||||
scheduler.wallclock().AndReturn(st + 1.5)
|
timeutils.wallclock().AndReturn(st + 1.5)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
@ -691,7 +692,7 @@ class TaskTest(common.HeatTestCase):
|
||||||
self.assertFalse(runner)
|
self.assertFalse(runner)
|
||||||
|
|
||||||
def test_timeout_swallowed(self):
|
def test_timeout_swallowed(self):
|
||||||
st = scheduler.wallclock()
|
st = timeutils.wallclock()
|
||||||
|
|
||||||
def task():
|
def task():
|
||||||
while True:
|
while True:
|
||||||
|
@ -701,10 +702,10 @@ class TaskTest(common.HeatTestCase):
|
||||||
yield
|
yield
|
||||||
self.fail('Task still running')
|
self.fail('Task still running')
|
||||||
|
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
scheduler.wallclock().AndReturn(st)
|
timeutils.wallclock().AndReturn(st)
|
||||||
scheduler.wallclock().AndReturn(st + 0.5)
|
timeutils.wallclock().AndReturn(st + 0.5)
|
||||||
scheduler.wallclock().AndReturn(st + 1.5)
|
timeutils.wallclock().AndReturn(st + 1.5)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
@ -777,21 +778,21 @@ class TaskTest(common.HeatTestCase):
|
||||||
self.assertTrue(runner.step())
|
self.assertTrue(runner.step())
|
||||||
|
|
||||||
def test_cancel_grace_period(self):
|
def test_cancel_grace_period(self):
|
||||||
st = scheduler.wallclock()
|
st = timeutils.wallclock()
|
||||||
task = DummyTask(5)
|
task = DummyTask(5)
|
||||||
|
|
||||||
self.m.StubOutWithMock(task, 'do_step')
|
self.m.StubOutWithMock(task, 'do_step')
|
||||||
self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
|
self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
|
|
||||||
task.do_step(1).AndReturn(None)
|
task.do_step(1).AndReturn(None)
|
||||||
task.do_step(2).AndReturn(None)
|
task.do_step(2).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st)
|
timeutils.wallclock().AndReturn(st)
|
||||||
scheduler.wallclock().AndReturn(st + 0.5)
|
timeutils.wallclock().AndReturn(st + 0.5)
|
||||||
task.do_step(3).AndReturn(None)
|
task.do_step(3).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 1.0)
|
timeutils.wallclock().AndReturn(st + 1.0)
|
||||||
task.do_step(4).AndReturn(None)
|
task.do_step(4).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 1.5)
|
timeutils.wallclock().AndReturn(st + 1.5)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
@ -808,24 +809,24 @@ class TaskTest(common.HeatTestCase):
|
||||||
self.assertTrue(runner.step())
|
self.assertTrue(runner.step())
|
||||||
|
|
||||||
def test_cancel_grace_period_before_timeout(self):
|
def test_cancel_grace_period_before_timeout(self):
|
||||||
st = scheduler.wallclock()
|
st = timeutils.wallclock()
|
||||||
task = DummyTask(5)
|
task = DummyTask(5)
|
||||||
|
|
||||||
self.m.StubOutWithMock(task, 'do_step')
|
self.m.StubOutWithMock(task, 'do_step')
|
||||||
self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
|
self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
|
|
||||||
scheduler.wallclock().AndReturn(st)
|
timeutils.wallclock().AndReturn(st)
|
||||||
scheduler.wallclock().AndReturn(st + 0.1)
|
timeutils.wallclock().AndReturn(st + 0.1)
|
||||||
task.do_step(1).AndReturn(None)
|
task.do_step(1).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 0.2)
|
timeutils.wallclock().AndReturn(st + 0.2)
|
||||||
task.do_step(2).AndReturn(None)
|
task.do_step(2).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 0.2)
|
timeutils.wallclock().AndReturn(st + 0.2)
|
||||||
scheduler.wallclock().AndReturn(st + 0.5)
|
timeutils.wallclock().AndReturn(st + 0.5)
|
||||||
task.do_step(3).AndReturn(None)
|
task.do_step(3).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 1.0)
|
timeutils.wallclock().AndReturn(st + 1.0)
|
||||||
task.do_step(4).AndReturn(None)
|
task.do_step(4).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 1.5)
|
timeutils.wallclock().AndReturn(st + 1.5)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
@ -842,24 +843,24 @@ class TaskTest(common.HeatTestCase):
|
||||||
self.assertTrue(runner.step())
|
self.assertTrue(runner.step())
|
||||||
|
|
||||||
def test_cancel_grace_period_after_timeout(self):
|
def test_cancel_grace_period_after_timeout(self):
|
||||||
st = scheduler.wallclock()
|
st = timeutils.wallclock()
|
||||||
task = DummyTask(5)
|
task = DummyTask(5)
|
||||||
|
|
||||||
self.m.StubOutWithMock(task, 'do_step')
|
self.m.StubOutWithMock(task, 'do_step')
|
||||||
self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
|
self.m.StubOutWithMock(scheduler.TaskRunner, '_sleep')
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
|
|
||||||
scheduler.wallclock().AndReturn(st)
|
timeutils.wallclock().AndReturn(st)
|
||||||
scheduler.wallclock().AndReturn(st + 0.1)
|
timeutils.wallclock().AndReturn(st + 0.1)
|
||||||
task.do_step(1).AndReturn(None)
|
task.do_step(1).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 0.2)
|
timeutils.wallclock().AndReturn(st + 0.2)
|
||||||
task.do_step(2).AndReturn(None)
|
task.do_step(2).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 0.2)
|
timeutils.wallclock().AndReturn(st + 0.2)
|
||||||
scheduler.wallclock().AndReturn(st + 0.5)
|
timeutils.wallclock().AndReturn(st + 0.5)
|
||||||
task.do_step(3).AndReturn(None)
|
task.do_step(3).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 1.0)
|
timeutils.wallclock().AndReturn(st + 1.0)
|
||||||
task.do_step(4).AndReturn(None)
|
task.do_step(4).AndReturn(None)
|
||||||
scheduler.wallclock().AndReturn(st + 1.5)
|
timeutils.wallclock().AndReturn(st + 1.5)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
|
|
@ -24,6 +24,7 @@ import six
|
||||||
from heat.common import context
|
from heat.common import context
|
||||||
from heat.common import exception
|
from heat.common import exception
|
||||||
from heat.common import template_format
|
from heat.common import template_format
|
||||||
|
from heat.common import timeutils
|
||||||
from heat.db import api as db_api
|
from heat.db import api as db_api
|
||||||
from heat.engine.clients.os import keystone
|
from heat.engine.clients.os import keystone
|
||||||
from heat.engine.clients.os import nova
|
from heat.engine.clients.os import nova
|
||||||
|
@ -955,7 +956,7 @@ class StackTest(common.HeatTestCase):
|
||||||
|
|
||||||
def test_stack_create_timeout(self):
|
def test_stack_create_timeout(self):
|
||||||
self.m.StubOutWithMock(scheduler.DependencyTaskGroup, '__call__')
|
self.m.StubOutWithMock(scheduler.DependencyTaskGroup, '__call__')
|
||||||
self.m.StubOutWithMock(scheduler, 'wallclock')
|
self.m.StubOutWithMock(timeutils, 'wallclock')
|
||||||
|
|
||||||
stk = stack.Stack(self.ctx, 's', self.tmpl)
|
stk = stack.Stack(self.ctx, 's', self.tmpl)
|
||||||
|
|
||||||
|
@ -964,10 +965,10 @@ class StackTest(common.HeatTestCase):
|
||||||
yield
|
yield
|
||||||
|
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
scheduler.wallclock().AndReturn(start_time)
|
timeutils.wallclock().AndReturn(start_time)
|
||||||
scheduler.wallclock().AndReturn(start_time + 1)
|
timeutils.wallclock().AndReturn(start_time + 1)
|
||||||
scheduler.DependencyTaskGroup.__call__().AndReturn(dummy_task())
|
scheduler.DependencyTaskGroup.__call__().AndReturn(dummy_task())
|
||||||
scheduler.wallclock().AndReturn(start_time + stk.timeout_secs() + 1)
|
timeutils.wallclock().AndReturn(start_time + stk.timeout_secs() + 1)
|
||||||
|
|
||||||
self.m.ReplayAll()
|
self.m.ReplayAll()
|
||||||
|
|
||||||
|
|
|
@ -22,6 +22,7 @@ import mock
|
||||||
from heat.common import exception
|
from heat.common import exception
|
||||||
from heat.common import heat_keystoneclient as hkc
|
from heat.common import heat_keystoneclient as hkc
|
||||||
from heat.common import template_format
|
from heat.common import template_format
|
||||||
|
from heat.common import timeutils
|
||||||
from heat.engine.clients.os import keystone
|
from heat.engine.clients.os import keystone
|
||||||
from heat.engine import scheduler
|
from heat.engine import scheduler
|
||||||
from heat.engine import stack
|
from heat.engine import stack
|
||||||
|
@ -379,7 +380,7 @@ class StackTest(common.HeatTestCase):
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
mock_tg = self.patchobject(scheduler.DependencyTaskGroup, '__call__',
|
mock_tg = self.patchobject(scheduler.DependencyTaskGroup, '__call__',
|
||||||
return_value=dummy_task())
|
return_value=dummy_task())
|
||||||
mock_wallclock = self.patchobject(scheduler, 'wallclock')
|
mock_wallclock = self.patchobject(timeutils, 'wallclock')
|
||||||
mock_wallclock.side_effect = [
|
mock_wallclock.side_effect = [
|
||||||
start_time,
|
start_time,
|
||||||
start_time + 1,
|
start_time + 1,
|
||||||
|
|
|
@ -46,6 +46,21 @@ class ISO8601UtilityTest(common.HeatTestCase):
|
||||||
self.assertRaises(ValueError, util.parse_isoduration, 'ABCDEFGH')
|
self.assertRaises(ValueError, util.parse_isoduration, 'ABCDEFGH')
|
||||||
|
|
||||||
|
|
||||||
|
class DurationTest(common.HeatTestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(DurationTest, self).setUp()
|
||||||
|
st = util.wallclock()
|
||||||
|
mock_clock = self.patchobject(util, 'wallclock')
|
||||||
|
mock_clock.side_effect = [st, st + 0.5]
|
||||||
|
|
||||||
|
def test_duration_not_expired(self):
|
||||||
|
self.assertFalse(util.Duration(1.0).expired())
|
||||||
|
|
||||||
|
def test_duration_expired(self):
|
||||||
|
self.assertTrue(util.Duration(0.1).expired())
|
||||||
|
|
||||||
|
|
||||||
class RetryBackoffExponentialTest(common.HeatTestCase):
|
class RetryBackoffExponentialTest(common.HeatTestCase):
|
||||||
|
|
||||||
scenarios = [(
|
scenarios = [(
|
||||||
|
|
Loading…
Reference in New Issue