Merge "Add way to group exceptions in DependencyTaskGroup"
This commit is contained in:
commit
bca680bc91
|
@ -128,7 +128,9 @@ class Graph(collections.defaultdict):
|
||||||
node = self[key]
|
node = self[key]
|
||||||
|
|
||||||
for src in node.required_by():
|
for src in node.required_by():
|
||||||
self[src] -= key
|
src_node = self[src]
|
||||||
|
if key in src_node:
|
||||||
|
src_node -= key
|
||||||
|
|
||||||
return super(Graph, self).__delitem__(key)
|
return super(Graph, self).__delitem__(key)
|
||||||
|
|
||||||
|
|
|
@ -75,6 +75,29 @@ class Timeout(BaseException):
|
||||||
return wallclock() > self._endtime
|
return wallclock() > self._endtime
|
||||||
|
|
||||||
|
|
||||||
|
class ExceptionGroup(Exception):
|
||||||
|
'''
|
||||||
|
Container for multiple exceptions.
|
||||||
|
|
||||||
|
This exception is used by DependencyTaskGroup when the flag
|
||||||
|
aggregate_exceptions is set to True and it's re-raised again when all tasks
|
||||||
|
are finished. This way it can be caught later on so that the individual
|
||||||
|
exceptions can be acted upon.
|
||||||
|
'''
|
||||||
|
|
||||||
|
def __init__(self, exceptions=None):
|
||||||
|
if exceptions is None:
|
||||||
|
exceptions = list()
|
||||||
|
|
||||||
|
self.exceptions = list(exceptions)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return str(map(str, self.exceptions))
|
||||||
|
|
||||||
|
def __unicode__(self):
|
||||||
|
return unicode(map(str, self.exceptions))
|
||||||
|
|
||||||
|
|
||||||
class TaskRunner(object):
|
class TaskRunner(object):
|
||||||
"""
|
"""
|
||||||
Wrapper for a resumable task (co-routine).
|
Wrapper for a resumable task (co-routine).
|
||||||
|
@ -182,11 +205,14 @@ class TaskRunner(object):
|
||||||
self._sleep(wait_time)
|
self._sleep(wait_time)
|
||||||
|
|
||||||
def cancel(self):
|
def cancel(self):
|
||||||
"""Cancel the task if it is running."""
|
"""Cancel the task and mark it as done."""
|
||||||
if self.started() and not self.done():
|
if not self.done():
|
||||||
logger.debug(_('%s cancelled') % str(self))
|
logger.debug(_('%s cancelled') % str(self))
|
||||||
self._runner.close()
|
try:
|
||||||
self._done = True
|
if self.started():
|
||||||
|
self._runner.close()
|
||||||
|
finally:
|
||||||
|
self._done = True
|
||||||
|
|
||||||
def started(self):
|
def started(self):
|
||||||
"""Return True if the task has been started."""
|
"""Return True if the task has been started."""
|
||||||
|
@ -268,7 +294,7 @@ class DependencyTaskGroup(object):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, dependencies, task=lambda o: o(),
|
def __init__(self, dependencies, task=lambda o: o(),
|
||||||
reverse=False, name=None):
|
reverse=False, name=None, aggregate_exceptions=False):
|
||||||
"""
|
"""
|
||||||
Initialise with the task dependencies and (optionally) a task to run on
|
Initialise with the task dependencies and (optionally) a task to run on
|
||||||
each.
|
each.
|
||||||
|
@ -276,9 +302,14 @@ class DependencyTaskGroup(object):
|
||||||
If no task is supplied, it is assumed that the tasks are stored
|
If no task is supplied, it is assumed that the tasks are stored
|
||||||
directly in the dependency tree. If a task is supplied, the object
|
directly in the dependency tree. If a task is supplied, the object
|
||||||
stored in the dependency tree is passed as an argument.
|
stored in the dependency tree is passed as an argument.
|
||||||
|
|
||||||
|
If aggregate_exceptions is set to True, then all the tasks will be run
|
||||||
|
and any raised exceptions will be stored to be re-raised after all
|
||||||
|
tasks are done.
|
||||||
"""
|
"""
|
||||||
self._runners = dict((o, TaskRunner(task, o)) for o in dependencies)
|
self._runners = dict((o, TaskRunner(task, o)) for o in dependencies)
|
||||||
self._graph = dependencies.graph(reverse=reverse)
|
self._graph = dependencies.graph(reverse=reverse)
|
||||||
|
self.aggregate_exceptions = aggregate_exceptions
|
||||||
|
|
||||||
if name is None:
|
if name is None:
|
||||||
name = '(%s) %s' % (getattr(task, '__name__',
|
name = '(%s) %s' % (getattr(task, '__name__',
|
||||||
|
@ -292,21 +323,40 @@ class DependencyTaskGroup(object):
|
||||||
|
|
||||||
def __call__(self):
|
def __call__(self):
|
||||||
"""Return a co-routine which runs the task group."""
|
"""Return a co-routine which runs the task group."""
|
||||||
|
raised_exceptions = []
|
||||||
try:
|
try:
|
||||||
while any(self._runners.itervalues()):
|
while any(self._runners.itervalues()):
|
||||||
for k, r in self._ready():
|
try:
|
||||||
r.start()
|
for k, r in self._ready():
|
||||||
|
r.start()
|
||||||
|
|
||||||
yield
|
yield
|
||||||
|
|
||||||
for k, r in self._running():
|
for k, r in self._running():
|
||||||
if r.step():
|
if r.step():
|
||||||
del self._graph[k]
|
del self._graph[k]
|
||||||
|
except Exception as e:
|
||||||
|
self._cancel_recursively(k, r)
|
||||||
|
if not self.aggregate_exceptions:
|
||||||
|
raise
|
||||||
|
raised_exceptions.append(e)
|
||||||
except:
|
except:
|
||||||
with excutils.save_and_reraise_exception():
|
with excutils.save_and_reraise_exception():
|
||||||
for r in self._runners.itervalues():
|
for r in self._runners.itervalues():
|
||||||
r.cancel()
|
r.cancel()
|
||||||
|
|
||||||
|
if raised_exceptions:
|
||||||
|
raise ExceptionGroup(raised_exceptions)
|
||||||
|
|
||||||
|
def _cancel_recursively(self, key, runner):
|
||||||
|
runner.cancel()
|
||||||
|
node = self._graph[key]
|
||||||
|
for dependent_node in node.required_by():
|
||||||
|
node_runner = self._runners[dependent_node]
|
||||||
|
self._cancel_recursively(dependent_node, node_runner)
|
||||||
|
|
||||||
|
del self._graph[key]
|
||||||
|
|
||||||
def _ready(self):
|
def _ready(self):
|
||||||
"""
|
"""
|
||||||
Iterate over all subtasks that are ready to start - i.e. all their
|
Iterate over all subtasks that are ready to start - i.e. all their
|
||||||
|
|
|
@ -163,11 +163,41 @@ class PollingTaskGroupTest(HeatTestCase):
|
||||||
dummy.do_step(1, i, i * i)
|
dummy.do_step(1, i, i * i)
|
||||||
|
|
||||||
|
|
||||||
class DependencyTaskGroupTest(HeatTestCase):
|
class ExceptionGroupTest(HeatTestCase):
|
||||||
|
|
||||||
|
def test_contains_exceptions(self):
|
||||||
|
exception_group = scheduler.ExceptionGroup()
|
||||||
|
self.assertIsInstance(exception_group.exceptions, list)
|
||||||
|
|
||||||
|
def test_can_be_initialized_with_a_list_of_exceptions(self):
|
||||||
|
ex1 = Exception("ex 1")
|
||||||
|
ex2 = Exception("ex 2")
|
||||||
|
|
||||||
|
exception_group = scheduler.ExceptionGroup([ex1, ex2])
|
||||||
|
self.assertIn(ex1, exception_group.exceptions)
|
||||||
|
self.assertIn(ex2, exception_group.exceptions)
|
||||||
|
|
||||||
|
def test_can_add_exceptions_after_init(self):
|
||||||
|
ex = Exception()
|
||||||
|
exception_group = scheduler.ExceptionGroup()
|
||||||
|
|
||||||
|
exception_group.exceptions.append(ex)
|
||||||
|
self.assertIn(ex, exception_group.exceptions)
|
||||||
|
|
||||||
|
def test_str_representation_aggregates_all_exceptions(self):
|
||||||
|
ex1 = Exception("ex 1")
|
||||||
|
ex2 = Exception("ex 2")
|
||||||
|
|
||||||
|
exception_group = scheduler.ExceptionGroup([ex1, ex2])
|
||||||
|
self.assertEqual("['ex 1', 'ex 2']", str(exception_group))
|
||||||
|
|
||||||
|
|
||||||
|
class DependencyTaskGroupTest(HeatTestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(DependencyTaskGroupTest, self).setUp()
|
super(DependencyTaskGroupTest, self).setUp()
|
||||||
self.addCleanup(self.m.VerifyAll)
|
self.addCleanup(self.m.VerifyAll)
|
||||||
|
self.aggregate_exceptions = False
|
||||||
|
self.reverse_order = False
|
||||||
|
|
||||||
@contextlib.contextmanager
|
@contextlib.contextmanager
|
||||||
def _dep_test(self, *edges):
|
def _dep_test(self, *edges):
|
||||||
|
@ -175,7 +205,9 @@ class DependencyTaskGroupTest(HeatTestCase):
|
||||||
|
|
||||||
deps = dependencies.Dependencies(edges)
|
deps = dependencies.Dependencies(edges)
|
||||||
|
|
||||||
tg = scheduler.DependencyTaskGroup(deps, dummy)
|
tg = scheduler.DependencyTaskGroup(
|
||||||
|
deps, dummy, aggregate_exceptions=self.aggregate_exceptions,
|
||||||
|
reverse=self.reverse_order)
|
||||||
|
|
||||||
self.m.StubOutWithMock(dummy, 'do_step')
|
self.m.StubOutWithMock(dummy, 'do_step')
|
||||||
|
|
||||||
|
@ -320,6 +352,54 @@ class DependencyTaskGroupTest(HeatTestCase):
|
||||||
self.assertRaises(dependencies.CircularDependencyException,
|
self.assertRaises(dependencies.CircularDependencyException,
|
||||||
scheduler.DependencyTaskGroup, d)
|
scheduler.DependencyTaskGroup, d)
|
||||||
|
|
||||||
|
def test_aggregate_exceptions_raises_all_at_the_end(self):
|
||||||
|
def run_tasks_with_exceptions(e1=None, e2=None):
|
||||||
|
self.aggregate_exceptions = True
|
||||||
|
tasks = (('A', None), ('B', None), ('C', None))
|
||||||
|
with self._dep_test(*tasks) as dummy:
|
||||||
|
dummy.do_step(1, 'A').InAnyOrder('1')
|
||||||
|
dummy.do_step(1, 'B').InAnyOrder('1')
|
||||||
|
dummy.do_step(1, 'C').InAnyOrder('1').AndRaise(e1)
|
||||||
|
|
||||||
|
dummy.do_step(2, 'A').InAnyOrder('2')
|
||||||
|
dummy.do_step(2, 'B').InAnyOrder('2').AndRaise(e2)
|
||||||
|
|
||||||
|
dummy.do_step(3, 'A').InAnyOrder('3')
|
||||||
|
|
||||||
|
e1 = Exception('e1')
|
||||||
|
e2 = Exception('e2')
|
||||||
|
|
||||||
|
exc = self.assertRaises(scheduler.ExceptionGroup,
|
||||||
|
run_tasks_with_exceptions, e1, e2)
|
||||||
|
self.assertEqual(set([e1, e2]), set(exc.exceptions))
|
||||||
|
|
||||||
|
def test_aggregate_exceptions_cancels_dependent_tasks_recursively(self):
|
||||||
|
def run_tasks_with_exceptions(e1=None, e2=None):
|
||||||
|
self.aggregate_exceptions = True
|
||||||
|
tasks = (('A', None), ('B', 'A'), ('C', 'B'))
|
||||||
|
with self._dep_test(*tasks) as dummy:
|
||||||
|
dummy.do_step(1, 'A').AndRaise(e1)
|
||||||
|
|
||||||
|
e1 = Exception('e1')
|
||||||
|
|
||||||
|
exc = self.assertRaises(scheduler.ExceptionGroup,
|
||||||
|
run_tasks_with_exceptions, e1)
|
||||||
|
self.assertEqual([e1], exc.exceptions)
|
||||||
|
|
||||||
|
def test_aggregate_exceptions_cancels_tasks_in_reverse_order(self):
|
||||||
|
def run_tasks_with_exceptions(e1=None, e2=None):
|
||||||
|
self.reverse_order = True
|
||||||
|
self.aggregate_exceptions = True
|
||||||
|
tasks = (('A', None), ('B', 'A'), ('C', 'B'))
|
||||||
|
with self._dep_test(*tasks) as dummy:
|
||||||
|
dummy.do_step(1, 'C').AndRaise(e1)
|
||||||
|
|
||||||
|
e1 = Exception('e1')
|
||||||
|
|
||||||
|
exc = self.assertRaises(scheduler.ExceptionGroup,
|
||||||
|
run_tasks_with_exceptions, e1)
|
||||||
|
self.assertEqual([e1], exc.exceptions)
|
||||||
|
|
||||||
|
|
||||||
class TaskTest(HeatTestCase):
|
class TaskTest(HeatTestCase):
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue