openstack/deb-python-taskflow
Code Issues Proposed changes
Files
a809bfe77f0c7ffb860edffd8d82f55c54227d49
deb-python-taskflow/taskflow/tests/unit/test_flow_dependencies.py
Anastasia Karpinska 75eac61a06 Flow smart revert with retry controller 
- Remove flow REVERTING state. Now flow can be running and reverting simultaneously.
  Until flow isn't finished it is in RUNNING state.

- Add RETRYING state for the retry controller.

- Implement smart revertion and flow retries and retries resumption.

- Default retry controllers: Times, ForEach and ParameterizedForEach.

- Example and unit tests.

Implements: blueprint subgraph-execution
Implements: blueprint reversion-strategies
Implements: blueprint smart-revert

Change-Id: Ifa600bcad1edf2910f02ac36783cd458afbd880c
2014-03-18 11:34:01 +02:00

456 lines
20 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from taskflow.patterns import graph_flow as gf
from taskflow.patterns import linear_flow as lf
from taskflow.patterns import unordered_flow as uf
from taskflow import exceptions
from taskflow import retry
from taskflow import test
from taskflow.tests import utils
class FlowDependenciesTest(test.TestCase):
def test_task_without_dependencies(self):
flow = utils.TaskNoRequiresNoReturns()
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_task_requires_default_values(self):
flow = utils.TaskMultiArg()
self.assertEqual(flow.requires, set(['x', 'y', 'z']))
self.assertEqual(flow.provides, set())
def test_task_requires_rebinded_mapped(self):
flow = utils.TaskMultiArg(rebind={'x': 'a', 'y': 'b', 'z': 'c'})
self.assertEqual(flow.requires, set(['a', 'b', 'c']))
self.assertEqual(flow.provides, set())
def test_task_requires_additional_values(self):
flow = utils.TaskMultiArg(requires=['a', 'b'])
self.assertEqual(flow.requires, set(['a', 'b', 'x', 'y', 'z']))
self.assertEqual(flow.provides, set())
def test_task_provides_values(self):
flow = utils.TaskMultiReturn(provides=['a', 'b', 'c'])
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['a', 'b', 'c']))
def test_task_provides_and_requires_values(self):
flow = utils.TaskMultiArgMultiReturn(provides=['a', 'b', 'c'])
self.assertEqual(flow.requires, set(['x', 'y', 'z']))
self.assertEqual(flow.provides, set(['a', 'b', 'c']))
def test_linear_flow_without_dependencies(self):
flow = lf.Flow('lf').add(
utils.TaskNoRequiresNoReturns('task1'),
utils.TaskNoRequiresNoReturns('task2'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_linear_flow_requires_values(self):
flow = lf.Flow('lf').add(
utils.TaskOneArg('task1'),
utils.TaskMultiArg('task2'))
self.assertEqual(flow.requires, set(['x', 'y', 'z']))
self.assertEqual(flow.provides, set())
def test_linear_flow_requires_rebind_values(self):
flow = lf.Flow('lf').add(
utils.TaskOneArg('task1', rebind=['q']),
utils.TaskMultiArg('task2'))
self.assertEqual(flow.requires, set(['x', 'y', 'z', 'q']))
self.assertEqual(flow.provides, set())
def test_linear_flow_provides_values(self):
flow = lf.Flow('lf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskMultiReturn('task2', provides=['a', 'b', 'c']))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x', 'a', 'b', 'c']))
def test_linear_flow_provides_out_of_order(self):
flow = lf.Flow('lf')
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneArg('task2'),
utils.TaskOneReturn('task1', provides='x'))
def test_linear_flow_provides_required_values(self):
flow = lf.Flow('lf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x']))
def test_linear_flow_multi_provides_and_requires_values(self):
flow = lf.Flow('lf').add(
utils.TaskMultiArgMultiReturn('task1',
rebind=['a', 'b', 'c'],
provides=['x', 'y', 'q']),
utils.TaskMultiArgMultiReturn('task2',
provides=['i', 'j', 'k']))
self.assertEqual(flow.requires, set(['a', 'b', 'c', 'z']))
self.assertEqual(flow.provides, set(['x', 'y', 'q', 'i', 'j', 'k']))
def test_linear_flow_provides_same_values(self):
flow = lf.Flow('lf').add(utils.TaskOneReturn(provides='x'))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn(provides='x'))
def test_linear_flow_provides_same_values_one_add(self):
flow = lf.Flow('lf')
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn(provides='x'),
utils.TaskOneReturn(provides='x'))
def test_unordered_flow_without_dependencies(self):
flow = uf.Flow('uf').add(
utils.TaskNoRequiresNoReturns('task1'),
utils.TaskNoRequiresNoReturns('task2'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_unordered_flow_requires_values(self):
flow = uf.Flow('uf').add(
utils.TaskOneArg('task1'),
utils.TaskMultiArg('task2'))
self.assertEqual(flow.requires, set(['x', 'y', 'z']))
self.assertEqual(flow.provides, set())
def test_unordered_flow_requires_rebind_values(self):
flow = uf.Flow('uf').add(
utils.TaskOneArg('task1', rebind=['q']),
utils.TaskMultiArg('task2'))
self.assertEqual(flow.requires, set(['x', 'y', 'z', 'q']))
self.assertEqual(flow.provides, set())
def test_unordered_flow_provides_values(self):
flow = uf.Flow('uf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskMultiReturn('task2', provides=['a', 'b', 'c']))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x', 'a', 'b', 'c']))
def test_unordered_flow_provides_required_values(self):
flow = uf.Flow('uf')
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
def test_unordered_flow_requires_provided_value_other_call(self):
flow = uf.Flow('uf')
flow.add(utils.TaskOneReturn('task1', provides='x'))
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneArg('task2'))
def test_unordered_flow_provides_required_value_other_call(self):
flow = uf.Flow('uf')
flow.add(utils.TaskOneArg('task2'))
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneReturn('task1', provides='x'))
def test_unordered_flow_multi_provides_and_requires_values(self):
flow = uf.Flow('uf').add(
utils.TaskMultiArgMultiReturn('task1',
rebind=['a', 'b', 'c'],
provides=['d', 'e', 'f']),
utils.TaskMultiArgMultiReturn('task2',
provides=['i', 'j', 'k']))
self.assertEqual(flow.requires, set(['a', 'b', 'c', 'x', 'y', 'z']))
self.assertEqual(flow.provides, set(['d', 'e', 'f', 'i', 'j', 'k']))
def test_unordered_flow_provides_same_values(self):
flow = uf.Flow('uf').add(utils.TaskOneReturn(provides='x'))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn(provides='x'))
def test_unordered_flow_provides_same_values_one_add(self):
flow = uf.Flow('uf')
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn(provides='x'),
utils.TaskOneReturn(provides='x'))
def test_nested_flows_requirements(self):
flow = uf.Flow('uf').add(
lf.Flow('lf').add(
utils.TaskOneArgOneReturn('task1',
rebind=['a'], provides=['x']),
utils.TaskOneArgOneReturn('task2', provides=['y'])),
uf.Flow('uf').add(
utils.TaskOneArgOneReturn('task3',
rebind=['b'], provides=['z']),
utils.TaskOneArgOneReturn('task4', rebind=['c'],
provides=['q'])))
self.assertEqual(flow.requires, set(['a', 'b', 'c']))
self.assertEqual(flow.provides, set(['x', 'y', 'z', 'q']))
def test_nested_flows_provides_same_values(self):
flow = lf.Flow('lf').add(
uf.Flow('uf').add(utils.TaskOneReturn(provides='x')))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
gf.Flow('gf').add(utils.TaskOneReturn(provides='x')))
def test_graph_flow_without_dependencies(self):
flow = gf.Flow('gf').add(
utils.TaskNoRequiresNoReturns('task1'),
utils.TaskNoRequiresNoReturns('task2'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_graph_flow_requires_values(self):
flow = gf.Flow('gf').add(
utils.TaskOneArg('task1'),
utils.TaskMultiArg('task2'))
self.assertEqual(flow.requires, set(['x', 'y', 'z']))
self.assertEqual(flow.provides, set())
def test_graph_flow_requires_rebind_values(self):
flow = gf.Flow('gf').add(
utils.TaskOneArg('task1', rebind=['q']),
utils.TaskMultiArg('task2'))
self.assertEqual(flow.requires, set(['x', 'y', 'z', 'q']))
self.assertEqual(flow.provides, set())
def test_graph_flow_provides_values(self):
flow = gf.Flow('gf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskMultiReturn('task2', provides=['a', 'b', 'c']))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x', 'a', 'b', 'c']))
def test_graph_flow_provides_required_values(self):
flow = gf.Flow('gf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x']))
def test_graph_flow_provides_provided_value_other_call(self):
flow = gf.Flow('gf')
flow.add(utils.TaskOneReturn('task1', provides='x'))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn('task2', provides='x'))
def test_graph_flow_multi_provides_and_requires_values(self):
flow = gf.Flow('gf').add(
utils.TaskMultiArgMultiReturn('task1',
rebind=['a', 'b', 'c'],
provides=['d', 'e', 'f']),
utils.TaskMultiArgMultiReturn('task2',
provides=['i', 'j', 'k']))
self.assertEqual(flow.requires, set(['a', 'b', 'c', 'x', 'y', 'z']))
self.assertEqual(flow.provides, set(['d', 'e', 'f', 'i', 'j', 'k']))
def test_graph_cyclic_dependency(self):
flow = gf.Flow('g-3-cyclic')
self.assertRaisesRegexp(exceptions.DependencyFailure, '^No path',
flow.add,
utils.TaskOneArgOneReturn(provides='a',
requires=['b']),
utils.TaskOneArgOneReturn(provides='b',
requires=['c']),
utils.TaskOneArgOneReturn(provides='c',
requires=['a']))
def test_task_requires_and_provides_same_values(self):
self.assertRaises(exceptions.InvariantViolation,
utils.TaskOneArgOneReturn,
requires='a',
provides='a')
def test_retry_in_linear_flow_no_requirements_no_provides(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_retry_in_linear_flow_with_requirements(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt', requires=['x', 'y']))
self.assertEqual(flow.requires, set(['x', 'y']))
self.assertEqual(flow.provides, set())
def test_retry_in_linear_flow_with_provides(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt', provides=['x', 'y']))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x', 'y']))
def test_retry_in_linear_flow_requires_and_provides(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt',
requires=['x', 'y'],
provides=['a', 'b']))
self.assertEqual(flow.requires, set(['x', 'y']))
self.assertEqual(flow.provides, set(['a', 'b']))
def test_retry_requires_and_provides_same_value(self):
self.assertRaises(exceptions.InvariantViolation,
retry.AlwaysRevert,
'rt', requires=['x', 'y'], provides=['x', 'y'])
def test_retry_in_unordered_flow_no_requirements_no_provides(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_retry_in_unordered_flow_with_requirements(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', requires=['x', 'y']))
self.assertEqual(flow.requires, set(['x', 'y']))
self.assertEqual(flow.provides, set())
def test_retry_in_unordered_flow_with_provides(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', provides=['x', 'y']))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x', 'y']))
def test_retry_in_unordered_flow_requires_and_provides(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt',
requires=['x', 'y'],
provides=['a', 'b']))
self.assertEqual(flow.requires, set(['x', 'y']))
self.assertEqual(flow.provides, set(['a', 'b']))
def test_retry_in_graph_flow_no_requirements_no_provides(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt'))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set())
def test_retry_in_graph_flow_with_requirements(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt', requires=['x', 'y']))
self.assertEqual(flow.requires, set(['x', 'y']))
self.assertEqual(flow.provides, set())
def test_retry_in_graph_flow_with_provides(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt', provides=['x', 'y']))
self.assertEqual(flow.requires, set())
self.assertEqual(flow.provides, set(['x', 'y']))
def test_retry_in_graph_flow_requires_and_provides(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt',
requires=['x', 'y'],
provides=['a', 'b']))
self.assertEqual(flow.requires, set(['x', 'y']))
self.assertEqual(flow.provides, set(['a', 'b']))
def test_linear_flow_retry_and_task(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt',
requires=['x', 'y'],
provides=['a', 'b']))
flow.add(utils.TaskMultiArgOneReturn(rebind=['a', 'x', 'c'],
provides=['z']))
self.assertEqual(flow.requires, set(['x', 'y', 'c']))
self.assertEqual(flow.provides, set(['a', 'b', 'z']))
def test_linear_flow_retry_and_task_dependency_conflict(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt', requires=['x']))
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneReturn(provides=['x']))
def test_linear_flow_retry_and_task_provide_same_value(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt', provides=['x']))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn('t1', provides=['x']))
def test_unordered_flow_retry_and_task(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt',
requires=['x', 'y'],
provides=['a', 'b']))
flow.add(utils.TaskMultiArgOneReturn(rebind=['a', 'x', 'c'],
provides=['z']))
self.assertEqual(flow.requires, set(['x', 'y', 'c']))
self.assertEqual(flow.provides, set(['a', 'b', 'z']))
def test_unordered_flow_retry_and_task_dependency_conflict(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', requires=['x']))
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneReturn(provides=['x']))
def test_unordered_flow_retry_and_task_provide_same_value(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', provides=['x']))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn('t1', provides=['x']))
def test_unordered_flow_retry_two_tasks_provide_same_value(self):
flow = uf.Flow('uf', retry.AlwaysRevert('rt', provides=['y']))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn('t1', provides=['x']),
utils.TaskOneReturn('t2', provides=['x']))
def test_graph_flow_retry_and_task(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt',
requires=['x', 'y'],
provides=['a', 'b']))
flow.add(utils.TaskMultiArgOneReturn(rebind=['a', 'x', 'c'],
provides=['z']))
self.assertEqual(flow.requires, set(['x', 'y', 'c']))
self.assertEqual(flow.provides, set(['a', 'b', 'z']))
def test_graph_flow_retry_and_task_dependency_conflict(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt', requires=['x']))
self.assertRaises(exceptions.InvariantViolation,
flow.add,
utils.TaskOneReturn(provides=['x']))
def test_graph_flow_retry_and_task_provide_same_value(self):
flow = gf.Flow('gf', retry.AlwaysRevert('rt', provides=['x']))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn('t1', provides=['x']))
def test_two_retries_provide_same_values_in_nested_flows(self):
flow = lf.Flow('lf', retry.AlwaysRevert('rt1', provides=['x']))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
lf.Flow('lf1', retry.AlwaysRevert('rt2',
provides=['x'])))
def test_two_retries_provide_same_values(self):
flow = lf.Flow('lf').add(
lf.Flow('lf1', retry.AlwaysRevert('rt1', provides=['x'])))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
lf.Flow('lf2', retry.AlwaysRevert('rt2',
provides=['x'])))
def test_builtin_retry_args(self):
class FullArgsRetry(retry.AlwaysRevert):
def execute(self, history, **kwargs):
pass
def revert(self, history, **kwargs):
pass
flow = lf.Flow('lf', retry=FullArgsRetry(requires='a'))
self.assertEqual(flow.requires, set(['a']))
View Git Blame Copy Permalink