openstack
/
taskflow
Code Issues Proposed changes

Fix up python 3.3 incompatabilities 

Make the python 3.3 testing work by selectively
disabling & including eventlet, switch to testtools
and testrepository which has 2.6, 2.7, 3.2+ unified
testing support so that we can correctly run our
tests in all supported python versions.

Closes-Bug: #1251660
Co-authored-by: Alexander Gorodnev <agorodnev@griddynamics.com>
Change-Id: I23b6f04387cfd3bf6b5a044edffa446ca897ce3a
This commit is contained in:
Joshua Harlow 2013-11-16 02:44:14 -08:00 committed by Alexander Gorodnev
parent 359ce523f4
commit db15db8186
32 changed files with 280 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.testr.conf Normal file
View File

@ -0,0 +1,9 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \
${PYTHON:-python} -m subunit.run discover -t ./ ./taskflow/tests $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

2
requirements.txt
View File

@ -15,7 +15,7 @@ stevedore>=0.10
# Backport for concurrent.futures which exists in 3.2+
futures>=2.1.3
# Only needed if the eventlet executor is used.
eventlet>=0.13.0
# eventlet>=0.13.0
# NOTE(harlowja): if you want to be able to use the graph_utils
# export_graph_to_dot function you will need to uncomment the following.
# pydot>=1.0

2
setup.cfg
View File

@ -17,6 +17,8 @@ classifier =
Programming Language :: Python :: 2
Programming Language :: Python :: 2.6
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
[global]
setup-hooks =

5
taskflow/examples/resume_vm_boot.py
View File

@ -81,7 +81,7 @@ def get_backend():
class PrintText(task.Task):
"""Just inserts some text print outs in a workflow."""
def __init__(self, print_what, no_slow=False):
content_hash = hashlib.md5(print_what).hexdigest()[0:8]
content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
super(PrintText, self).__init__(name="Print: %s" % (content_hash))
self._text = print_what
self._no_slow = no_slow
@ -257,8 +257,9 @@ except (IndexError, ValueError):
# Set up how we want our engine to run, serial, parallel...
engine_conf = {
'engine': 'parallel',
'executor': e_utils.GreenExecutor(5),
}
if e_utils.EVENTLET_AVAILABLE:
engine_conf['executor'] = e_utils.GreenExecutor(5)
# Create/fetch a logbook that will track the workflows work.
book = None

11
taskflow/examples/resume_volume_create.py
View File

@ -38,7 +38,6 @@ from taskflow import engines
from taskflow import task
from taskflow.persistence import backends
from taskflow.utils import eventlet_utils as e_utils
from taskflow.utils import persistence_utils as p_utils
@ -83,7 +82,7 @@ def get_backend():
class PrintText(task.Task):
def __init__(self, print_what, no_slow=False):
content_hash = hashlib.md5(print_what).hexdigest()[0:8]
content_hash = hashlib.md5(print_what.encode('utf-8')).hexdigest()[0:8]
super(PrintText, self).__init__(name="Print: %s" % (content_hash))
self._text = print_what
self._no_slow = no_slow
@ -156,13 +155,13 @@ else:
flow_detail = find_flow_detail(backend, book_id, flow_id)
# Annnnd load and run.
engine_conf = {
'engine': 'serial',
}
engine = engines.load(flow,
flow_detail=flow_detail,
backend=backend,
engine_conf={
'engine': 'parallel',
'executor': e_utils.GreenExecutor(10),
})
engine_conf=engine_conf)
engine.run()
# How to use.

2
taskflow/examples/reverting_linear.out.txt
View File

@ -2,4 +2,4 @@ Calling jim 555.
Calling joe 444.
Calling 444 and apologizing.
Calling 555 and apologizing.
Flow failed: IOError('Suzzie not home right now.',)
Flow failed: Suzzie not home right now.

2
taskflow/examples/reverting_linear.py
View File

@ -98,4 +98,4 @@ except Exception as e:
# You will also note that this is not a problem in this case since no
# parallelism is involved; this is ensured by the usage of a linear flow,
# which runs serially as well as the default engine type which is 'serial'.
print("Flow failed: %r" % e)
print("Flow failed: %s" % e)

4
taskflow/examples/simple_linear_listening.out.txt
View File

@ -1,10 +1,10 @@
Flow => RUNNING
Task __main__.call_jim => RUNNING
Calling jim.
Context = {'joe_number': 444, 'jim_number': 555}
Context = [('jim_number', 555), ('joe_number', 444)]
Task __main__.call_jim => SUCCESS
Task __main__.call_joe => RUNNING
Calling joe.
Context = {'joe_number': 444, 'jim_number': 555}
Context = [('jim_number', 555), ('joe_number', 444)]
Task __main__.call_joe => SUCCESS
Flow => SUCCESS

4
taskflow/examples/simple_linear_listening.py
View File

@ -58,12 +58,12 @@ from taskflow import task
def call_jim(context):
print("Calling jim.")
print("Context = %s" % (context))
print("Context = %s" % (sorted(context.items(), key=lambda x: x[0])))
def call_joe(context):
print("Calling joe.")
print("Context = %s" % (context))
print("Context = %s" % (sorted(context.items(), key=lambda x: x[0])))
def flow_watch(state, details):

5
taskflow/persistence/backends/impl_sqlalchemy.py
View File

@ -36,6 +36,7 @@ from taskflow.persistence.backends import base
from taskflow.persistence.backends.sqlalchemy import migration
from taskflow.persistence.backends.sqlalchemy import models
from taskflow.persistence import logbook
from taskflow.utils import eventlet_utils
from taskflow.utils import misc
from taskflow.utils import persistence_utils
@ -224,7 +225,9 @@ class SQLAlchemyBackend(base.Backend):
# or engine arg overrides make sure we merge them in.
engine_args.update(conf.pop('engine_args', {}))
engine = sa.create_engine(sql_connection, **engine_args)
if misc.as_bool(conf.pop('checkin_yield', True)):
checkin_yield = conf.pop('checkin_yield',
eventlet_utils.EVENTLET_AVAILABLE)
if misc.as_bool(checkin_yield):
sa.event.listen(engine, 'checkin', _thread_yield)
if 'mysql' in e_url.drivername:
if misc.as_bool(conf.pop('checkout_ping', True)):

38
taskflow/test.py
View File

@ -16,17 +16,41 @@
# License for the specific language governing permissions and limitations
# under the License.
import unittest2
from testtools import compat
from testtools import matchers
from testtools import testcase
class TestCase(unittest2.TestCase):
"""Test case base class for all unit tests."""
class TestCase(testcase.TestCase):
"""Test case base class for all taskflow unit tests."""
def setUp(self):
super(TestCase, self).setUp()
def assertRaisesRegexp(self, exc_class, pattern, callable_obj,
*args, **kwargs):
# TODO(harlowja): submit a pull/review request to testtools to add
# this method to there codebase instead of having it exist in ours
# since it really doesn't belong here.
def tearDown(self):
super(TestCase, self).tearDown()
class ReRaiseOtherTypes(object):
def match(self, matchee):
if not issubclass(matchee[0], exc_class):
compat.reraise(*matchee)
class CaptureMatchee(object):
def match(self, matchee):
self.matchee = matchee[1]
capture = CaptureMatchee()
matcher = matchers.Raises(matchers.MatchesAll(ReRaiseOtherTypes(),
matchers.MatchesException(exc_class,
pattern),
capture))
our_callable = testcase.Nullary(callable_obj, *args, **kwargs)
self.assertThat(our_callable, matcher)
return capture.matchee
def assertRegexpMatches(self, text, pattern):
matcher = matchers.MatchesRegex(pattern)
self.assertThat(text, matcher)
def assertIsSubset(self, super_set, sub_set, msg=None):
missing_set = set()

2
taskflow/tests/test_examples.py
View File

@ -98,7 +98,7 @@ class ExamplesTestCase(taskflow.test.TestCase):
# replace them with some constant string
output = self.uuid_re.sub('<SOME UUID>', output)
expected_output = self.uuid_re.sub('<SOME UUID>', expected_output)
self.assertMultiLineEqual(output, expected_output)
self.assertEqual(output, expected_output)
ExamplesTestCase.update()

1
taskflow/tests/unit/persistence/test_memory_persistence.py
View File

@ -23,6 +23,7 @@ from taskflow.tests.unit.persistence import base
class MemoryPersistenceTest(test.TestCase, base.PersistenceTestMixin):
def setUp(self):
super(MemoryPersistenceTest, self).setUp()
self._backend = impl_memory.MemoryBackend({})
def _get_connection(self):

71
taskflow/tests/unit/test_action_engine.py
View File

@ -85,30 +85,34 @@ class EngineTaskTest(utils.EngineTestBase):
'fail reverted(Failure: RuntimeError: Woot!)',
'fail REVERTED',
'flow REVERTED']
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(self.values, expected)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
now_expected = expected + ['fail PENDING', 'flow PENDING'] + expected
self.assertEqual(self.values, now_expected)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
def test_invalid_flow_raises(self):
value = 'i am string, not task/flow, sorry'
with self.assertRaises(TypeError) as err:
def compile_bad(value):
engine = self._make_engine(value)
engine.compile()
self.assertIn(value, str(err.exception))
value = 'i am string, not task/flow, sorry'
err = self.assertRaises(TypeError, compile_bad, value)
self.assertIn(value, str(err))
def test_invalid_flow_raises_from_run(self):
value = 'i am string, not task/flow, sorry'
with self.assertRaises(TypeError) as err:
def run_bad(value):
engine = self._make_engine(value)
engine.run()
self.assertIn(value, str(err.exception))
value = 'i am string, not task/flow, sorry'
err = self.assertRaises(TypeError, run_bad, value)
self.assertIn(value, str(err))
class EngineLinearFlowTest(utils.EngineTestBase):
@ -136,8 +140,7 @@ class EngineLinearFlowTest(utils.EngineTestBase):
utils.FailingTask(name='fail')
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.fetch_all(), {})
def test_sequential_flow_nested_blocks(self):
@ -156,8 +159,7 @@ class EngineLinearFlowTest(utils.EngineTestBase):
utils.FailingTask(name='fail')
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
def test_revert_not_run_task_is_not_reverted(self):
flow = lf.Flow('revert-not-run').add(
@ -165,8 +167,7 @@ class EngineLinearFlowTest(utils.EngineTestBase):
utils.NeverRunningTask(),
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(
self.values,
['fail reverted(Failure: RuntimeError: Woot!)'])
@ -180,8 +181,7 @@ class EngineLinearFlowTest(utils.EngineTestBase):
)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(
self.values,
['task1', 'task2',
@ -195,7 +195,7 @@ class EngineLinearFlowTest(utils.EngineTestBase):
def revert(m_self, result, flow_failures):
self.assertEqual(result, 'RESULT')
self.assertEqual(flow_failures.keys(), ['fail1'])
self.assertEqual(list(flow_failures.keys()), ['fail1'])
fail = flow_failures['fail1']
self.assertIsInstance(fail, misc.Failure)
self.assertEqual(str(fail), 'Failure: RuntimeError: Woot!')
@ -205,8 +205,7 @@ class EngineLinearFlowTest(utils.EngineTestBase):
utils.FailingTask(self.values, 'fail1')
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
class EngineParallelFlowTest(utils.EngineTestBase):
@ -236,8 +235,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
utils.TaskNoRequiresNoReturns(name='task2')
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
def test_parallel_revert_exception_is_reraised(self):
# NOTE(imelnikov): if we put NastyTask and FailingTask
@ -252,8 +250,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
utils.FailingTask(self.values, sleep=0.1)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
def test_sequential_flow_two_tasks_with_resumption(self):
flow = lf.Flow('lf-2-r').add(
@ -285,8 +282,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
utils.SaveOrderTask(self.values, name='task2', sleep=0.01)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
result = set(self.values)
# NOTE(harlowja): task 1/2 may or may not have executed, even with the
# sleeps due to the fact that the above is an unordered flow.
@ -303,8 +299,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
name='task2') # this should not get reverted
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
result = set(self.values)
self.assertEqual(result, set(['task1']))
@ -319,8 +314,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
result = set(self.values)
# Task1, task2 may *not* have executed and also may have *not* reverted
# since the above is an unordered flow so take that into account by
@ -381,8 +375,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
result = set(self.values)
# Task3 may or may not have executed, depending on scheduling and
# task ordering selection, so it may or may not exist in the result set
@ -407,8 +400,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
result = set(self.values)
# Since this is an unordered flow we can not guarantee that task1 or
# task2 will exist and be reverted, although they may exist depending
@ -432,8 +424,7 @@ class EngineParallelFlowTest(utils.EngineTestBase):
)
)
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
result = set(self.values)
possible_result = set(['task1', 'task1 reverted(5)',
'task2', 'task2 reverted(5)',
@ -492,8 +483,7 @@ class EngineGraphFlowTest(utils.EngineTestBase):
utils.SaveOrderTask(self.values, name='task1', provides='a'))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(
self.values,
['task1', 'task2',
@ -508,8 +498,7 @@ class EngineGraphFlowTest(utils.EngineTestBase):
utils.SaveOrderTask(self.values, name='task1', provides='a'))
engine = self._make_engine(flow)
with self.assertRaisesRegexp(RuntimeError, '^Gotcha'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Gotcha', engine.run)
self.assertEqual(engine.storage.get_flow_state(), states.FAILURE)
def test_graph_flow_with_multireturn_and_multiargs_tasks(self):

19
taskflow/tests/unit/test_arguments_passing.py
View File

@ -61,8 +61,9 @@ class ArgumentsPassingTest(utils.EngineTestBase):
})
def test_bad_save_as_value(self):
with self.assertRaises(TypeError):
utils.TaskOneReturn(name='task1', provides=object())
self.assertRaises(TypeError,
utils.TaskOneReturn,
name='task1', provides=object())
def test_arguments_passing(self):
flow = utils.TaskMultiArgOneReturn(provides='result')
@ -78,8 +79,7 @@ class ArgumentsPassingTest(utils.EngineTestBase):
flow = utils.TaskMultiArg()
engine = self._make_engine(flow)
engine.storage.inject({'a': 1, 'b': 4, 'x': 17})
with self.assertRaises(exc.MissingDependencies):
engine.run()
self.assertRaises(exc.MissingDependencies, engine.run)
def test_partial_arguments_mapping(self):
flow = utils.TaskMultiArgOneReturn(provides='result',
@ -109,19 +109,18 @@ class ArgumentsPassingTest(utils.EngineTestBase):
flow = utils.TaskMultiArg(rebind={'z': 'b'})
engine = self._make_engine(flow)
engine.storage.inject({'a': 1, 'y': 4, 'c': 9, 'x': 17})
with self.assertRaises(exc.MissingDependencies):
engine.run()
self.assertRaises(exc.MissingDependencies, engine.run)
def test_invalid_argument_name_list(self):
flow = utils.TaskMultiArg(rebind=['a', 'z', 'b'])
engine = self._make_engine(flow)
engine.storage.inject({'a': 1, 'b': 4, 'c': 9, 'x': 17})
with self.assertRaises(exc.MissingDependencies):
engine.run()
self.assertRaises(exc.MissingDependencies, engine.run)
def test_bad_rebind_args_value(self):
with self.assertRaises(TypeError):
utils.TaskOneArg(rebind=object())
self.assertRaises(TypeError,
utils.TaskOneArg,
rebind=object())
class SingleThreadedEngineTest(ArgumentsPassingTest,

7
taskflow/tests/unit/test_check_transition.py
View File

@ -40,6 +40,7 @@ class CheckFlowTransitionTest(test.TestCase):
states.check_flow_transition(states.RUNNING, states.RESUMING))
def test_bad_transition_raises(self):
with self.assertRaisesRegexp(exc.InvalidStateException,
'^Flow transition.*not allowed'):
states.check_flow_transition(states.FAILURE, states.SUCCESS)
self.assertRaisesRegexp(exc.InvalidStateException,
'^Flow transition.*not allowed',
states.check_flow_transition,
states.FAILURE, states.SUCCESS)

30
taskflow/tests/unit/test_engine_helpers.py
View File

@ -29,25 +29,28 @@ class FlowFromDetailTestCase(test.TestCase):
def test_no_meta(self):
_lb, flow_detail = p_utils.temporary_flow_detail()
self.assertIs(flow_detail.meta, None)
expected_msg = '^Cannot .* no factory information saved.$'
with self.assertRaisesRegexp(ValueError, expected_msg):
taskflow.engines.flow_from_detail(flow_detail)
self.assertRaisesRegexp(ValueError,
'^Cannot .* no factory information saved.$',
taskflow.engines.flow_from_detail,
flow_detail)
def test_no_factory_in_meta(self):
_lb, flow_detail = p_utils.temporary_flow_detail()
flow_detail.meta = {}
expected_msg = '^Cannot .* no factory information saved.$'
with self.assertRaisesRegexp(ValueError, expected_msg):
taskflow.engines.flow_from_detail(flow_detail)
self.assertRaisesRegexp(ValueError,
'^Cannot .* no factory information saved.$',
taskflow.engines.flow_from_detail,
flow_detail)
def test_no_importable_function(self):
_lb, flow_detail = p_utils.temporary_flow_detail()
flow_detail.meta = dict(factory=dict(
name='you can not import me, i contain spaces'
))
expected_msg = '^Could not import factory'
with self.assertRaisesRegexp(ImportError, expected_msg):
taskflow.engines.flow_from_detail(flow_detail)
self.assertRaisesRegexp(ImportError,
'^Could not import factory',
taskflow.engines.flow_from_detail,
flow_detail)
def test_no_arg_factory(self):
name = 'some.test.factory'
@ -79,11 +82,14 @@ def my_flow_factory(task_name):
class LoadFromFactoryTestCase(test.TestCase):
def test_non_reimportable(self):
def factory():
pass
with self.assertRaisesRegexp(ValueError,
'Flow factory .* is not reimportable'):
taskflow.engines.load_from_factory(factory)
self.assertRaisesRegexp(ValueError,
'Flow factory .* is not reimportable',
taskflow.engines.load_from_factory,
factory)
def test_it_works(self):
engine = taskflow.engines.load_from_factory(

12
taskflow/tests/unit/test_flattening.py
View File

@ -174,14 +174,14 @@ class FlattenTest(test.TestCase):
t_utils.DummyTask(name="a"),
t_utils.DummyTask(name="a")
)
with self.assertRaisesRegexp(exc.InvariantViolationException,
'^Tasks with duplicate names'):
f_utils.flatten(flo)
self.assertRaisesRegexp(exc.InvariantViolationException,
'^Tasks with duplicate names',
f_utils.flatten, flo)
def test_flatten_checks_for_dups_globally(self):
flo = gf.Flow("test").add(
gf.Flow("int1").add(t_utils.DummyTask(name="a")),
gf.Flow("int2").add(t_utils.DummyTask(name="a")))
with self.assertRaisesRegexp(exc.InvariantViolationException,
'^Tasks with duplicate names'):
f_utils.flatten(flo)
self.assertRaisesRegexp(exc.InvariantViolationException,
'^Tasks with duplicate names',
f_utils.flatten, flo)

67
taskflow/tests/unit/test_flow_dependencies.py
View File

@ -86,10 +86,11 @@ class FlowDependenciesTest(test.TestCase):
self.assertEqual(flow.provides, set(['x', 'a', 'b', 'c']))
def test_linear_flow_provides_out_of_order(self):
with self.assertRaises(exceptions.InvariantViolationException):
lf.Flow('lf').add(
utils.TaskOneArg('task2'),
utils.TaskOneReturn('task1', provides='x'))
flow = lf.Flow('lf')
self.assertRaises(exceptions.InvariantViolationException,
flow.add,
utils.TaskOneArg('task2'),
utils.TaskOneReturn('task1', provides='x'))
def test_linear_flow_provides_required_values(self):
flow = lf.Flow('lf').add(
@ -110,8 +111,10 @@ class FlowDependenciesTest(test.TestCase):
def test_linear_flow_self_requires(self):
flow = lf.Flow('uf')
with self.assertRaises(exceptions.InvariantViolationException):
flow.add(utils.TaskNoRequiresNoReturns(rebind=['x'], provides='x'))
self.assertRaises(exceptions.InvariantViolationException,
flow.add,
utils.TaskNoRequiresNoReturns(rebind=['x'],
provides='x'))
def test_unordered_flow_without_dependencies(self):
flow = uf.Flow('uf').add(
@ -122,8 +125,10 @@ class FlowDependenciesTest(test.TestCase):
def test_unordered_flow_self_requires(self):
flow = uf.Flow('uf')
with self.assertRaises(exceptions.InvariantViolationException):
flow.add(utils.TaskNoRequiresNoReturns(rebind=['x'], provides='x'))
self.assertRaises(exceptions.InvariantViolationException,
flow.add,
utils.TaskNoRequiresNoReturns(rebind=['x'],
provides='x'))
def test_unordered_flow_reuires_values(self):
flow = uf.Flow('uf').add(
@ -147,22 +152,25 @@ class FlowDependenciesTest(test.TestCase):
self.assertEqual(flow.provides, set(['x', 'a', 'b', 'c']))
def test_unordered_flow_provides_required_values(self):
with self.assertRaises(exceptions.InvariantViolationException):
uf.Flow('uf').add(
utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
flow = uf.Flow('uf')
self.assertRaises(exceptions.InvariantViolationException,
flow.add,
utils.TaskOneReturn('task1', provides='x'),
utils.TaskOneArg('task2'))
def test_unordered_flow_requires_provided_value_other_call(self):
flow = uf.Flow('uf')
flow.add(utils.TaskOneReturn('task1', provides='x'))
with self.assertRaises(exceptions.InvariantViolationException):
flow.add(utils.TaskOneArg('task2'))
self.assertRaises(exceptions.InvariantViolationException,
flow.add,
utils.TaskOneArg('task2'))
def test_unordered_flow_provides_required_value_other_call(self):
flow = uf.Flow('uf')
flow.add(utils.TaskOneArg('task2'))
with self.assertRaises(exceptions.InvariantViolationException):
flow.add(utils.TaskOneReturn('task1', provides='x'))
self.assertRaises(exceptions.InvariantViolationException,
flow.add,
utils.TaskOneReturn('task1', provides='x'))
def test_unordered_flow_multi_provides_and_requires_values(self):
flow = uf.Flow('uf').add(
@ -196,9 +204,11 @@ class FlowDependenciesTest(test.TestCase):
self.assertEqual(flow.provides, set())
def test_graph_flow_self_requires(self):
with self.assertRaisesRegexp(exceptions.DependencyFailure, '^No path'):
gf.Flow('g-1-req-error').add(
utils.TaskOneArgOneReturn(requires=['a'], provides='a'))
flow = gf.Flow('g-1-req-error')
self.assertRaisesRegexp(exceptions.DependencyFailure, '^No path',
flow.add,
utils.TaskOneArgOneReturn(requires=['a'],
provides='a'))
def test_graph_flow_reuires_values(self):
flow = gf.Flow('gf').add(
@ -231,8 +241,9 @@ class FlowDependenciesTest(test.TestCase):
def test_graph_flow_provides_provided_value_other_call(self):
flow = gf.Flow('gf')
flow.add(utils.TaskOneReturn('task1', provides='x'))
with self.assertRaises(exceptions.DependencyFailure):
flow.add(utils.TaskOneReturn('task2', provides='x'))
self.assertRaises(exceptions.DependencyFailure,
flow.add,
utils.TaskOneReturn('task2', provides='x'))
def test_graph_flow_multi_provides_and_requires_values(self):
flow = gf.Flow('gf').add(
@ -245,8 +256,12 @@ class FlowDependenciesTest(test.TestCase):
self.assertEqual(flow.provides, set(['d', 'e', 'f', 'i', 'j', 'k']))
def test_graph_cyclic_dependency(self):
with self.assertRaisesRegexp(exceptions.DependencyFailure, '^No path'):
gf.Flow('g-3-cyclic').add(
utils.TaskOneArgOneReturn(provides='a', requires=['b']),
utils.TaskOneArgOneReturn(provides='b', requires=['c']),
utils.TaskOneArgOneReturn(provides='c', requires=['a']))
flow = gf.Flow('g-3-cyclic')
self.assertRaisesRegexp(exceptions.DependencyFailure, '^No path',
flow.add,
utils.TaskOneArgOneReturn(provides='a',
requires=['b']),
utils.TaskOneArgOneReturn(provides='b',
requires=['c']),
utils.TaskOneArgOneReturn(provides='c',
requires=['a']))

4
taskflow/tests/unit/test_functor_task.py
View File

@ -63,6 +63,6 @@ class FunctorTaskTest(test.TestCase):
t(bof.run_one, revert=bof.revert_one),
t(bof.run_fail)
)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
taskflow.engines.run(flow)
self.assertRaisesRegexp(RuntimeError, '^Woot',
taskflow.engines.run, flow)
self.assertEqual(values, ['one', 'fail', 'revert one'])

4
taskflow/tests/unit/test_green_executor.py
View File

@ -19,11 +19,13 @@
import collections
import functools
from taskflow import test
import testtools
from taskflow import test
from taskflow.utils import eventlet_utils as eu
@testtools.skipIf(not eu.EVENTLET_AVAILABLE, 'eventlet is not available')
class GreenExecutorTest(test.TestCase):
def make_funcs(self, called, amount):

50
taskflow/tests/unit/test_storage.py
View File

@ -124,8 +124,7 @@ class StorageTest(test.TestCase):
def test_get_non_existing_var(self):
s = self._get_storage()
s.add_task('42', 'my task')
with self.assertRaises(exceptions.NotFound):
s.get('42')
self.assertRaises(exceptions.NotFound, s.get, '42')
def test_reset(self):
s = self._get_storage()
@ -133,8 +132,7 @@ class StorageTest(test.TestCase):
s.save('42', 5)
s.reset('42')
self.assertEqual(s.get_task_state('42'), states.PENDING)
with self.assertRaises(exceptions.NotFound):
s.get('42')
self.assertRaises(exceptions.NotFound, s.get, '42')
def test_reset_unknown_task(self):
s = self._get_storage()
@ -151,11 +149,9 @@ class StorageTest(test.TestCase):
s.reset_tasks()
self.assertEqual(s.get_task_state('42'), states.PENDING)
with self.assertRaises(exceptions.NotFound):
s.get('42')
self.assertRaises(exceptions.NotFound, s.get, '42')
self.assertEqual(s.get_task_state('43'), states.PENDING)
with self.assertRaises(exceptions.NotFound):
s.get('43')
self.assertRaises(exceptions.NotFound, s.get, '43')
def test_reset_tasks_does_not_breaks_inject(self):
s = self._get_storage()
@ -182,9 +178,9 @@ class StorageTest(test.TestCase):
def test_fetch_unknown_name(self):
s = self._get_storage()
with self.assertRaisesRegexp(exceptions.NotFound,
"^Name 'xxx' is not mapped"):
s.fetch('xxx')
self.assertRaisesRegexp(exceptions.NotFound,
"^Name 'xxx' is not mapped",
s.fetch, 'xxx')
def test_default_task_progress(self):
s = self._get_storage()
@ -230,8 +226,7 @@ class StorageTest(test.TestCase):
s.add_task('42', 'my task')
name = 'my result'
s.set_result_mapping('42', {name: None})
with self.assertRaises(exceptions.NotFound):
s.get(name)
self.assertRaises(exceptions.NotFound, s.get, name)
self.assertEqual(s.fetch_all(), {})
def test_save_multiple_results(self):
@ -297,8 +292,8 @@ class StorageTest(test.TestCase):
def test_fetch_not_found_args(self):
s = self._get_storage()
s.inject({'foo': 'bar', 'spam': 'eggs'})
with self.assertRaises(exceptions.NotFound):
s.fetch_mapped_args({'viking': 'helmet'})
self.assertRaises(exceptions.NotFound,
s.fetch_mapped_args, {'viking': 'helmet'})
def test_set_and_get_task_state(self):
s = self._get_storage()
@ -309,8 +304,8 @@ class StorageTest(test.TestCase):
def test_get_state_of_unknown_task(self):
s = self._get_storage()
with self.assertRaisesRegexp(exceptions.NotFound, '^Unknown'):
s.get_task_state('42')
self.assertRaisesRegexp(exceptions.NotFound, '^Unknown',
s.get_task_state, '42')
def test_task_by_name(self):
s = self._get_storage()
@ -319,9 +314,9 @@ class StorageTest(test.TestCase):
def test_unknown_task_by_name(self):
s = self._get_storage()
with self.assertRaisesRegexp(exceptions.NotFound,
'^Unknown task name:'):
s.get_uuid_by_name('42')
self.assertRaisesRegexp(exceptions.NotFound,
'^Unknown task name:',
s.get_uuid_by_name, '42')
def test_initial_flow_state(self):
s = self._get_storage()
@ -348,9 +343,8 @@ class StorageTest(test.TestCase):
s.save('42', {})
mocked_warning.assert_called_once_with(
mock.ANY, 'my task', 'key', 'result')
with self.assertRaisesRegexp(exceptions.NotFound,
'^Unable to find result'):
s.fetch('result')
self.assertRaisesRegexp(exceptions.NotFound,
'^Unable to find result', s.fetch, 'result')
@mock.patch.object(storage.LOG, 'warning')
def test_empty_result_is_checked(self, mocked_warning):
@ -360,9 +354,8 @@ class StorageTest(test.TestCase):
s.save('42', ())
mocked_warning.assert_called_once_with(
mock.ANY, 'my task', 0, 'a')
with self.assertRaisesRegexp(exceptions.NotFound,
'^Unable to find result'):
s.fetch('a')
self.assertRaisesRegexp(exceptions.NotFound,
'^Unable to find result', s.fetch, 'a')
@mock.patch.object(storage.LOG, 'warning')
def test_short_result_is_checked(self, mocked_warning):
@ -373,9 +366,8 @@ class StorageTest(test.TestCase):
mocked_warning.assert_called_once_with(
mock.ANY, 'my task', 1, 'b')
self.assertEqual(s.fetch('a'), 'result')
with self.assertRaisesRegexp(exceptions.NotFound,
'^Unable to find result'):
s.fetch('b')
self.assertRaisesRegexp(exceptions.NotFound,
'^Unable to find result', s.fetch, 'b')
@mock.patch.object(storage.LOG, 'warning')
def test_multiple_providers_are_checked(self, mocked_warning):

12
taskflow/tests/unit/test_suspend_flow.py
View File

@ -132,8 +132,7 @@ class SuspendFlowTest(utils.EngineTestBase):
['a', 'b',
'c reverted(Failure: RuntimeError: Woot!)',
'b reverted(5)'])
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine.run)
self.assertEqual(engine.storage.get_flow_state(), states.REVERTED)
self.assertEqual(
self.values,
@ -155,8 +154,7 @@ class SuspendFlowTest(utils.EngineTestBase):
# pretend we are resuming
engine2 = self._make_engine(flow, engine.storage._flowdetail)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine2.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run)
self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED)
self.assertEqual(
self.values,
@ -182,8 +180,7 @@ class SuspendFlowTest(utils.EngineTestBase):
AutoSuspendingTaskOnRevert(self.values, 'b')
)
engine2 = self._make_engine(flow2, engine.storage._flowdetail)
with self.assertRaisesRegexp(RuntimeError, '^Woot'):
engine2.run()
self.assertRaisesRegexp(RuntimeError, '^Woot', engine2.run)
self.assertEqual(engine2.storage.get_flow_state(), states.REVERTED)
self.assertEqual(
self.values,
@ -207,8 +204,7 @@ class SuspendFlowTest(utils.EngineTestBase):
engine.storage.get_uuid_by_name(engine.storage.injector_name),
None,
states.FAILURE)
with self.assertRaises(exc.MissingDependencies):
engine.run()
self.assertRaises(exc.MissingDependencies, engine.run)
class SingleThreadedEngineTest(SuspendFlowTest,

21
taskflow/tests/unit/test_task.py
View File

@ -71,8 +71,8 @@ class TaskTestCase(test.TestCase):
self.assertEqual(my_task.save_as, {'food': 0})
def test_bad_provides(self):
with self.assertRaisesRegexp(TypeError, '^Task provides'):
MyTask(provides=object())
self.assertRaisesRegexp(TypeError, '^Task provides',
MyTask, provides=object())
def test_requires_by_default(self):
my_task = MyTask()
@ -100,8 +100,9 @@ class TaskTestCase(test.TestCase):
})
def test_requires_explicit_not_enough(self):
with self.assertRaisesRegexp(ValueError, '^Missing arguments'):
MyTask(auto_extract=False, requires=('spam', 'eggs'))
self.assertRaisesRegexp(ValueError, '^Missing arguments',
MyTask,
auto_extract=False, requires=('spam', 'eggs'))
def test_requires_ignores_optional(self):
my_task = DefaultArgTask()
@ -128,8 +129,8 @@ class TaskTestCase(test.TestCase):
})
def test_rebind_unknown(self):
with self.assertRaisesRegexp(ValueError, '^Extra arguments'):
MyTask(rebind={'foo': 'bar'})
self.assertRaisesRegexp(ValueError, '^Extra arguments',
MyTask, rebind={'foo': 'bar'})
def test_rebind_unknown_kwargs(self):
task = KwargsTask(rebind={'foo': 'bar'})
@ -155,8 +156,8 @@ class TaskTestCase(test.TestCase):
})
def test_rebind_list_more(self):
with self.assertRaisesRegexp(ValueError, '^Extra arguments'):
MyTask(rebind=('a', 'b', 'c', 'd'))
self.assertRaisesRegexp(ValueError, '^Extra arguments',
MyTask, rebind=('a', 'b', 'c', 'd'))
def test_rebind_list_more_kwargs(self):
task = KwargsTask(rebind=('a', 'b', 'c'))
@ -167,8 +168,8 @@ class TaskTestCase(test.TestCase):
})
def test_rebind_list_bad_value(self):
with self.assertRaisesRegexp(TypeError, '^Invalid rebind value:'):
MyTask(rebind=object())
self.assertRaisesRegexp(TypeError, '^Invalid rebind value:',
MyTask, rebind=object())
def test_default_provides(self):
task = DefaultProvidesTask()

14
taskflow/tests/unit/test_utils.py
View File

@ -227,13 +227,16 @@ class AttrDictTest(test.TestCase):
self.assertEqual(attrs, dict(obj))
def test_runtime_invalid_set(self):
def bad_assign(obj):
obj._123 = 'b'
attrs = {
'a': 1,
}
obj = misc.AttrDict(**attrs)
self.assertEqual(obj.a, 1)
with self.assertRaises(AttributeError):
obj._123 = 'b'
self.assertRaises(AttributeError, bad_assign, obj)
def test_bypass_get(self):
attrs = {
@ -243,14 +246,17 @@ class AttrDictTest(test.TestCase):
self.assertEqual(1, obj['a'])
def test_bypass_set_no_get(self):
def bad_assign(obj):
obj._b = 'e'
attrs = {
'a': 1,
}
obj = misc.AttrDict(**attrs)
self.assertEqual(1, obj['a'])
obj['_b'] = 'c'
with self.assertRaises(AttributeError):
obj._b = 'e'
self.assertRaises(AttributeError, bad_assign, obj)
self.assertEqual('c', obj['_b'])

31
taskflow/tests/unit/test_utils_failure.py
View File

@ -75,8 +75,7 @@ class CaptureFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin):
self.assertIs(exc_info[1], self.fail_obj.exception)
def test_reraises(self):
with self.assertRaisesRegexp(RuntimeError, '^Woot!$'):
self.fail_obj.reraise()
self.assertRaisesRegexp(RuntimeError, '^Woot!$', self.fail_obj.reraise)
class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin):
@ -95,9 +94,8 @@ class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin):
self.assertIs(self.fail_obj.exc_info, None)
def test_reraises(self):
with self.assertRaises(exceptions.WrappedFailure) as ctx:
self.fail_obj.reraise()
exc = ctx.exception
exc = self.assertRaises(exceptions.WrappedFailure,
self.fail_obj.reraise)
self.assertIs(exc.check(RuntimeError), RuntimeError)
@ -110,31 +108,30 @@ class FailureObjectTestCase(test.TestCase):
self.assertRaises(TypeError, misc.Failure)
def test_unknown_argument(self):
with self.assertRaises(TypeError) as ctx:
misc.Failure(
exception_str='Woot!',
traceback_str=None,
exc_type_names=['Exception'],
hi='hi there')
exc = self.assertRaises(TypeError, misc.Failure,
exception_str='Woot!',
traceback_str=None,
exc_type_names=['Exception'],
hi='hi there')
expected = "Failure.__init__ got unexpected keyword argument(s): hi"
self.assertEqual(str(ctx.exception), expected)
self.assertEqual(str(exc), expected)
def test_empty_does_not_reraise(self):
self.assertIs(misc.Failure.reraise_if_any([]), None)
def test_reraises_one(self):
fls = [_captured_failure('Woot!')]
with self.assertRaisesRegexp(RuntimeError, '^Woot!$'):
misc.Failure.reraise_if_any(fls)
self.assertRaisesRegexp(RuntimeError, '^Woot!$',
misc.Failure.reraise_if_any, fls)
def test_reraises_several(self):
fls = [
_captured_failure('Woot!'),
_captured_failure('Oh, not again!')
]
with self.assertRaises(exceptions.WrappedFailure) as ctx:
misc.Failure.reraise_if_any(fls)
self.assertEqual(list(ctx.exception), fls)
exc = self.assertRaises(exceptions.WrappedFailure,
misc.Failure.reraise_if_any, fls)
self.assertEqual(list(exc), fls)
def test_failure_copy(self):
fail_obj = _captured_failure('Woot!')

14
taskflow/utils/eventlet_utils.py
View File

@ -19,11 +19,14 @@
import logging
import threading
from eventlet.green import threading as gthreading
from eventlet import greenpool
from eventlet import patcher
from eventlet import queue
try:
from eventlet.green import threading as gthreading
from eventlet import greenpool
from eventlet import patcher
from eventlet import queue
EVENTLET_AVAILABLE = True
except ImportError:
EVENTLET_AVAILABLE = False
from concurrent import futures
@ -93,6 +96,7 @@ class GreenExecutor(futures.Executor):
"""A greenthread backed executor."""
def __init__(self, max_workers=1000):
assert EVENTLET_AVAILABLE, 'eventlet is needed to use GreenExecutor'
assert int(max_workers) > 0, 'Max workers must be greater than zero'
self._max_workers = int(max_workers)
self._pool = greenpool.GreenPool(self._max_workers)

26
taskflow/utils/misc.py
View File

@ -32,6 +32,7 @@ from taskflow.utils import reflection
LOG = logging.getLogger(__name__)
NUMERIC_TYPES = tuple(list(six.integer_types) + [float])
def wraps(fn):
@ -81,19 +82,29 @@ def is_valid_attribute_name(name, allow_self=False, allow_hidden=False):
return False
# Make the name just be a simple string in latin-1 encoding in python3
name = six.b(name)
if not allow_self and name.lower().startswith('self'):
if not allow_self and name.lower().startswith(six.b('self')):
return False
if not allow_hidden and name.startswith("_"):
if not allow_hidden and name.startswith(six.b("_")):
return False
# See: http://docs.python.org/release/2.5.2/ref/grammar.txt (or newer)
#
# Python identifiers should start with a letter.
if not name[0].isalpha():
return False
for i in range(1, len(name)):
# The rest of an attribute name follows: (letter | digit | "_")*
if not (name[i].isalpha() or name[i].isdigit() or name[i] == "_"):
if isinstance(name[0], six.integer_types):
if not chr(name[0]).isalpha():
return False
else:
if not name[0].isalpha():
return False
for i in range(1, len(name)):
symbol = name[i]
# The rest of an attribute name follows: (letter | digit | "_")*
if isinstance(symbol, six.integer_types):
symbol = chr(symbol)
if not (symbol.isalpha() or symbol.isdigit() or symbol == "_"):
return False
else:
if not (symbol.isalpha() or symbol.isdigit() or symbol == "_"):
return False
return True
@ -108,7 +119,6 @@ class AttrDict(dict):
if not is_valid_attribute_name(name):
return False
# Make the name just be a simple string in latin-1 encoding in python3
name = six.b(name)
if name in cls.NO_ATTRS:
return False
return True

2
taskflow/utils/persistence_utils.py
View File

@ -207,7 +207,7 @@ def _format_meta(metadata, indent):
for (k, v) in metadata.items():
# Progress for now is a special snowflake and will be formatted
# in percent format.
if k == 'progress' and isinstance(v, (float, int, long)):
if k == 'progress' and isinstance(v, misc.NUMERIC_TYPES):
v = "%0.2f%%" % (v * 100.0)
lines.append("%s+ %s = %s" % (" " * (indent + 2), k, v))
return lines

1
test-2.x-requirements.txt Normal file
View File

@ -0,0 +1 @@
eventlet>=0.13.0

15
test-requirements.txt
View File

@ -1,15 +1,6 @@
# Install bounded pep8/pyflakes first, then let flake8 install
pep8==1.4.5
pyflakes>=0.7.2,<0.7.4
flake8==2.0
hacking>=0.5.6,<0.8
discover
coverage>=3.6
mock>=1.0
nose
nose-exclude
openstack.nose_plugin>=0.7
pylint==0.25.2
# Needed for features in 2.7 not in 2.6
unittest2
testrepository>=0.0.17
testtools>=0.9.32

20
tox.ini
View File

@ -10,15 +10,19 @@ setenv = VIRTUAL_ENV={envdir}
LANG=en_US.UTF-8
LANGUAGE=en_US:en
LC_ALL=C
NOSE_WITH_OPENSTACK=1
NOSE_OPENSTACK_COLOR=1
NOSE_OPENSTACK_RED=0.05
NOSE_OPENSTACK_YELLOW=0.025
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands = nosetests {posargs}
commands = python setup.py testr --slowest --testr-args='{posargs}'
[testenv:py26]
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/test-2.x-requirements.txt
[testenv:py27]
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
-r{toxinidir}/test-2.x-requirements.txt
[tox:jenkins]
downloadcache = ~/cache/pip
@ -34,7 +38,7 @@ deps = -r{toxinidir}/requirements.txt
commands = pylint
[testenv:cover]
setenv = NOSE_WITH_COVERAGE=1
commands = python setup.py testr --coverage --testr-args='{posargs}'
[testenv:venv]
commands = {posargs}